nnetsauce
1from .base.base import Base 2from .base.baseRegressor import BaseRegressor 3from .boosting.adaBoostClassifier import AdaBoostClassifier 4from .custom.customClassifier import CustomClassifier 5from .custom.customRegressor import CustomRegressor 6from .deep.deepClassifier import DeepClassifier 7from .deep.deepRegressor import DeepRegressor 8from .deep.deepMTS import DeepMTS 9from .glm.glmClassifier import GLMClassifier 10from .glm.glmRegressor import GLMRegressor 11from .lazypredict.lazyClassifier import LazyClassifier 12from .lazypredict.lazyRegressor import LazyRegressor 13from .lazypredict.lazyMTS import LazyMTS 14from .lazypredict.lazydeepClassifier import LazyDeepClassifier 15from .lazypredict.lazydeepRegressor import LazyDeepRegressor 16from .lazypredict.lazydeepMTS import LazyDeepMTS 17from .mts.mts import MTS 18from .multitask.multitaskClassifier import MultitaskClassifier 19from .multitask.simplemultitaskClassifier import SimpleMultitaskClassifier 20from .optimizers.optimizer import Optimizer 21from .randombag.randomBagClassifier import RandomBagClassifier 22from .randombag.randomBagRegressor import RandomBagRegressor 23from .ridge2.ridge2Classifier import Ridge2Classifier 24from .ridge2.ridge2Regressor import Ridge2Regressor 25from .ridge2.ridge2MultitaskClassifier import Ridge2MultitaskClassifier 26from .rvfl.bayesianrvflRegressor import BayesianRVFLRegressor 27from .rvfl.bayesianrvfl2Regressor import BayesianRVFL2Regressor 28from .sampling import SubSampler 29 30__all__ = [ 31 "AdaBoostClassifier", 32 "Base", 33 "BaseRegressor", 34 "BayesianRVFLRegressor", 35 "BayesianRVFL2Regressor", 36 "CustomClassifier", 37 "CustomRegressor", 38 "DeepClassifier", 39 "DeepRegressor", 40 "DeepMTS", 41 "GLMClassifier", 42 "GLMRegressor", 43 "LazyClassifier", 44 "LazyRegressor", 45 "LazyMTS", 46 "LazyDeepClassifier", 47 "LazyDeepRegressor", 48 "LazyDeepMTS", 49 "MTS", 50 "MultitaskClassifier", 51 "SimpleMultitaskClassifier", 52 "Optimizer", 53 "RandomBagRegressor", 54 "RandomBagClassifier", 55 "Ridge2Regressor", 56 "Ridge2Classifier", 57 "Ridge2MultitaskClassifier", 58 "SubSampler", 59]
19class AdaBoostClassifier(Boosting, ClassifierMixin): 20 """AdaBoost Classification (SAMME) model class derived from class Boosting 21 22 Parameters: 23 24 obj: object 25 any object containing a method fit (obj.fit()) and a method predict 26 (obj.predict()) 27 28 n_estimators: int 29 number of boosting iterations 30 31 learning_rate: float 32 learning rate of the boosting procedure 33 34 n_hidden_features: int 35 number of nodes in the hidden layer 36 37 reg_lambda: float 38 regularization parameter for weights 39 40 reg_alpha: float 41 controls compromize between l1 and l2 norm of weights 42 43 activation_name: str 44 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 45 46 a: float 47 hyperparameter for 'prelu' or 'elu' activation function 48 49 nodes_sim: str 50 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 51 'uniform' 52 53 bias: boolean 54 indicates if the hidden layer contains a bias term (True) or not 55 (False) 56 57 dropout: float 58 regularization parameter; (random) percentage of nodes dropped out 59 of the training 60 61 direct_link: boolean 62 indicates if the original predictors are included (True) in model's 63 fitting or not (False) 64 65 n_clusters: int 66 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: 67 no clustering) 68 69 cluster_encode: bool 70 defines how the variable containing clusters is treated (default is one-hot) 71 if `False`, then labels are used, without one-hot encoding 72 73 type_clust: str 74 type of clustering method: currently k-means ('kmeans') or Gaussian 75 Mixture Model ('gmm') 76 77 type_scaling: a tuple of 3 strings 78 scaling methods for inputs, hidden layer, and clustering respectively 79 (and when relevant). 80 Currently available: standardization ('std') or MinMax scaling ('minmax') 81 82 col_sample: float 83 percentage of covariates randomly chosen for training 84 85 row_sample: float 86 percentage of rows chosen for training, by stratified bootstrapping 87 88 seed: int 89 reproducibility seed for nodes_sim=='uniform' 90 91 method: str 92 type of Adaboost method, 'SAMME' (discrete) or 'SAMME.R' (real) 93 94 backend: str 95 "cpu" or "gpu" or "tpu" 96 97 Attributes: 98 99 alpha_: list 100 AdaBoost coefficients alpha_m 101 102 base_learners_: dict 103 a dictionary containing the base learners 104 105 Examples: 106 107 See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py) 108 109 ```python 110 import nnetsauce as ns 111 import numpy as np 112 from sklearn.datasets import load_breast_cancer 113 from sklearn.linear_model import LogisticRegression 114 from sklearn.model_selection import train_test_split 115 from sklearn import metrics 116 from time import time 117 118 breast_cancer = load_breast_cancer() 119 Z = breast_cancer.data 120 t = breast_cancer.target 121 np.random.seed(123) 122 X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2) 123 124 # SAMME.R 125 clf = LogisticRegression(solver='liblinear', multi_class = 'ovr', 126 random_state=123) 127 fit_obj = ns.AdaBoostClassifier(clf, 128 n_hidden_features=int(11.22338867), 129 direct_link=True, 130 n_estimators=250, learning_rate=0.01126343, 131 col_sample=0.72684326, row_sample=0.86429443, 132 dropout=0.63078613, n_clusters=2, 133 type_clust="gmm", 134 verbose=1, seed = 123, 135 method="SAMME.R") 136 137 start = time() 138 fit_obj.fit(X_train, y_train) 139 print(f"Elapsed {time() - start}") 140 141 start = time() 142 print(fit_obj.score(X_test, y_test)) 143 print(f"Elapsed {time() - start}") 144 145 preds = fit_obj.predict(X_test) 146 147 print(fit_obj.score(X_test, y_test, scoring="roc_auc")) 148 print(metrics.classification_report(preds, y_test)) 149 150 ``` 151 152 """ 153 154 # construct the object ----- 155 156 def __init__( 157 self, 158 obj, 159 n_estimators=10, 160 learning_rate=0.1, 161 n_hidden_features=1, 162 reg_lambda=0, 163 reg_alpha=0.5, 164 activation_name="relu", 165 a=0.01, 166 nodes_sim="sobol", 167 bias=True, 168 dropout=0, 169 direct_link=False, 170 n_clusters=2, 171 cluster_encode=True, 172 type_clust="kmeans", 173 type_scaling=("std", "std", "std"), 174 col_sample=1, 175 row_sample=1, 176 seed=123, 177 verbose=1, 178 method="SAMME", 179 backend="cpu", 180 ): 181 super().__init__( 182 obj=obj, 183 n_estimators=n_estimators, 184 learning_rate=learning_rate, 185 n_hidden_features=n_hidden_features, 186 activation_name=activation_name, 187 a=a, 188 nodes_sim=nodes_sim, 189 bias=bias, 190 dropout=dropout, 191 direct_link=direct_link, 192 n_clusters=n_clusters, 193 cluster_encode=cluster_encode, 194 type_clust=type_clust, 195 type_scaling=type_scaling, 196 col_sample=col_sample, 197 row_sample=row_sample, 198 seed=seed, 199 backend=backend, 200 ) 201 202 self.type_fit = "classification" 203 self.verbose = verbose 204 self.method = method 205 self.reg_lambda = reg_lambda 206 self.reg_alpha = reg_alpha 207 208 self.alpha_ = [] 209 self.base_learners_ = dict.fromkeys(range(n_estimators)) 210 211 def fit(self, X, y, sample_weight=None, **kwargs): 212 """Fit Boosting model to training data (X, y). 213 214 Parameters: 215 216 X: {array-like}, shape = [n_samples, n_features] 217 Training vectors, where n_samples is the number 218 of samples and n_features is the number of features. 219 220 y: array-like, shape = [n_samples] 221 Target values. 222 223 **kwargs: additional parameters to be passed to 224 self.cook_training_set or self.obj.fit 225 226 Returns: 227 228 self: object 229 """ 230 231 assert mx.is_factor(y), "y must contain only integers" 232 233 assert self.method in ( 234 "SAMME", 235 "SAMME.R", 236 ), "`method` must be either 'SAMME' or 'SAMME.R'" 237 238 assert (self.reg_lambda <= 1) & ( 239 self.reg_lambda >= 0 240 ), "must have self.reg_lambda <= 1 & self.reg_lambda >= 0" 241 242 assert (self.reg_alpha <= 1) & ( 243 self.reg_alpha >= 0 244 ), "must have self.reg_alpha <= 1 & self.reg_alpha >= 0" 245 246 # training 247 n, p = X.shape 248 self.n_classes = len(np.unique(y)) 249 250 if sample_weight is None: 251 w_m = np.repeat(1.0 / n, n) 252 253 else: 254 w_m = np.asarray(sample_weight) 255 256 base_learner = CustomClassifier( 257 self.obj, 258 n_hidden_features=self.n_hidden_features, 259 activation_name=self.activation_name, 260 a=self.a, 261 nodes_sim=self.nodes_sim, 262 bias=self.bias, 263 dropout=self.dropout, 264 direct_link=self.direct_link, 265 n_clusters=self.n_clusters, 266 type_clust=self.type_clust, 267 type_scaling=self.type_scaling, 268 col_sample=self.col_sample, 269 row_sample=self.row_sample, 270 seed=self.seed, 271 ) 272 273 if self.verbose == 1: 274 pbar = Progbar(self.n_estimators) 275 276 if self.method == "SAMME": 277 err_m = 1e6 278 err_bound = 1 - 1 / self.n_classes 279 self.alpha_.append(1.0) 280 x_range_n = range(n) 281 282 for m in range(self.n_estimators): 283 preds = base_learner.fit( 284 X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs 285 ).predict(X) 286 287 self.base_learners_.update( 288 {m: pickle.loads(pickle.dumps(base_learner, -1))} 289 ) 290 291 cond = [y[i] != preds[i] for i in x_range_n] 292 293 err_m = max( 294 sum([elt[0] * elt[1] for elt in zip(cond, w_m)]), 295 2.220446049250313e-16, 296 ) # sum(w_m) == 1 297 298 if self.reg_lambda > 0: 299 err_m += self.reg_lambda * ( 300 (1 - self.reg_alpha) * 0.5 * sum([x**2 for x in w_m]) 301 + self.reg_alpha * sum([abs(x) for x in w_m]) 302 ) 303 304 err_m = min(err_m, err_bound) 305 306 alpha_m = self.learning_rate * log( 307 (self.n_classes - 1) * (1 - err_m) / err_m 308 ) 309 310 self.alpha_.append(alpha_m) 311 312 w_m_temp = [exp(alpha_m * cond[i]) for i in x_range_n] 313 314 sum_w_m = sum(w_m_temp) 315 316 w_m = np.asarray([w_m_temp[i] / sum_w_m for i in x_range_n]) 317 318 base_learner.set_params(seed=self.seed + (m + 1) * 1000) 319 320 if self.verbose == 1: 321 pbar.update(m) 322 323 if self.verbose == 1: 324 pbar.update(self.n_estimators) 325 326 self.n_estimators = len(self.base_learners_) 327 328 return self 329 330 if self.method == "SAMME.R": 331 Y = mo.one_hot_encode2(y, self.n_classes) 332 333 if sample_weight is None: 334 w_m = np.repeat(1.0 / n, n) # (N, 1) 335 336 else: 337 w_m = np.asarray(sample_weight) 338 339 for m in range(self.n_estimators): 340 probs = base_learner.fit( 341 X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs 342 ).predict_proba(X) 343 344 np.clip( 345 a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs 346 ) 347 348 self.base_learners_.update( 349 {m: pickle.loads(pickle.dumps(base_learner, -1))} 350 ) 351 352 w_m *= np.exp( 353 -1.0 354 * self.learning_rate 355 * (1.0 - 1.0 / self.n_classes) 356 * xlogy(Y, probs).sum(axis=1) 357 ) 358 359 w_m /= np.sum(w_m) 360 361 base_learner.set_params(seed=self.seed + (m + 1) * 1000) 362 363 if self.verbose == 1: 364 pbar.update(m) 365 366 if self.verbose == 1: 367 pbar.update(self.n_estimators) 368 369 self.n_estimators = len(self.base_learners_) 370 371 return self 372 373 def predict(self, X, **kwargs): 374 """Predict test data X. 375 376 Parameters: 377 378 X: {array-like}, shape = [n_samples, n_features] 379 Training vectors, where n_samples is the number 380 of samples and n_features is the number of features. 381 382 **kwargs: additional parameters to be passed to 383 self.cook_test_set 384 385 Returns: 386 387 model predictions: {array-like} 388 """ 389 390 return self.predict_proba(X, **kwargs).argmax(axis=1) 391 392 def predict_proba(self, X, **kwargs): 393 """Predict probabilities for test data X. 394 395 Parameters: 396 397 X: {array-like}, shape = [n_samples, n_features] 398 Training vectors, where n_samples is the number 399 of samples and n_features is the number of features. 400 401 **kwargs: additional parameters to be passed to 402 self.cook_test_set 403 404 Returns: 405 406 probability estimates for test data: {array-like} 407 408 """ 409 410 n_iter = len(self.base_learners_) 411 412 if self.method == "SAMME": 413 ensemble_learner = np.zeros((X.shape[0], self.n_classes)) 414 415 # if self.verbose == 1: 416 # pbar = Progbar(n_iter) 417 418 for idx, base_learner in self.base_learners_.items(): 419 preds = base_learner.predict(X, **kwargs) 420 421 ensemble_learner += self.alpha_[idx] * mo.one_hot_encode2( 422 preds, self.n_classes 423 ) 424 425 # if self.verbose == 1: 426 # pbar.update(idx) 427 428 # if self.verbose == 1: 429 # pbar.update(n_iter) 430 431 expit_ensemble_learner = expit(ensemble_learner) 432 433 sum_ensemble = expit_ensemble_learner.sum(axis=1) 434 435 return expit_ensemble_learner / sum_ensemble[:, None] 436 437 # if self.method == "SAMME.R": 438 ensemble_learner = 0 439 440 # if self.verbose == 1: 441 # pbar = Progbar(n_iter) 442 443 for idx, base_learner in self.base_learners_.items(): 444 probs = base_learner.predict_proba(X, **kwargs) 445 446 np.clip(a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs) 447 448 log_preds_proba = np.log(probs) 449 450 ensemble_learner += ( 451 log_preds_proba - log_preds_proba.mean(axis=1)[:, None] 452 ) 453 454 # if self.verbose == 1: 455 # pbar.update(idx) 456 457 ensemble_learner *= self.n_classes - 1 458 459 # if self.verbose == 1: 460 # pbar.update(n_iter) 461 462 expit_ensemble_learner = expit(ensemble_learner) 463 464 sum_ensemble = expit_ensemble_learner.sum(axis=1) 465 466 return expit_ensemble_learner / sum_ensemble[:, None] 467 468 def score(self, X, y, scoring=None, **kwargs): 469 """Score the model on test set features X and response y. 470 471 Parameters: 472 473 X: {array-like}, shape = [n_samples, n_features] 474 Training vectors, where n_samples is the number 475 of samples and n_features is the number of features 476 477 y: array-like, shape = [n_samples] 478 Target values 479 480 scoring: str 481 must be in ('accuracy', 'average_precision', 482 'brier_score_loss', 'f1', 'f1_micro', 483 'f1_macro', 'f1_weighted', 'f1_samples', 484 'neg_log_loss', 'precision', 'recall', 485 'roc_auc') 486 487 **kwargs: additional parameters to be passed to scoring functions 488 489 Returns: 490 491 model scores: {array-like} 492 493 """ 494 495 preds = self.predict(X) 496 497 if scoring is None: 498 scoring = "accuracy" 499 500 # check inputs 501 assert scoring in ( 502 "accuracy", 503 "average_precision", 504 "brier_score_loss", 505 "f1", 506 "f1_micro", 507 "f1_macro", 508 "f1_weighted", 509 "f1_samples", 510 "neg_log_loss", 511 "precision", 512 "recall", 513 "roc_auc", 514 ), "'scoring' should be in ('accuracy', 'average_precision', \ 515 'brier_score_loss', 'f1', 'f1_micro', \ 516 'f1_macro', 'f1_weighted', 'f1_samples', \ 517 'neg_log_loss', 'precision', 'recall', \ 518 'roc_auc')" 519 520 scoring_options = { 521 "accuracy": skm2.accuracy_score, 522 "average_precision": skm2.average_precision_score, 523 "brier_score_loss": skm2.brier_score_loss, 524 "f1": skm2.f1_score, 525 "f1_micro": skm2.f1_score, 526 "f1_macro": skm2.f1_score, 527 "f1_weighted": skm2.f1_score, 528 "f1_samples": skm2.f1_score, 529 "neg_log_loss": skm2.log_loss, 530 "precision": skm2.precision_score, 531 "recall": skm2.recall_score, 532 "roc_auc": skm2.roc_auc_score, 533 } 534 535 return scoring_options[scoring](y, preds, **kwargs)
AdaBoost Classification (SAMME) model class derived from class Boosting
Parameters:
obj: object
any object containing a method fit (obj.fit()) and a method predict
(obj.predict())
n_estimators: int
number of boosting iterations
learning_rate: float
learning rate of the boosting procedure
n_hidden_features: int
number of nodes in the hidden layer
reg_lambda: float
regularization parameter for weights
reg_alpha: float
controls compromize between l1 and l2 norm of weights
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not
(False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original predictors are included (True) in model's
fitting or not (False)
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
col_sample: float
percentage of covariates randomly chosen for training
row_sample: float
percentage of rows chosen for training, by stratified bootstrapping
seed: int
reproducibility seed for nodes_sim=='uniform'
method: str
type of Adaboost method, 'SAMME' (discrete) or 'SAMME.R' (real)
backend: str
"cpu" or "gpu" or "tpu"
Attributes:
alpha_: list
AdaBoost coefficients alpha_m
base_learners_: dict
a dictionary containing the base learners
Examples:
See also https://github.com/Techtonique/nnetsauce/blob/master/examples/adaboost_classification.py
import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time
breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
# SAMME.R
clf = LogisticRegression(solver='liblinear', multi_class = 'ovr',
random_state=123)
fit_obj = ns.AdaBoostClassifier(clf,
n_hidden_features=int(11.22338867),
direct_link=True,
n_estimators=250, learning_rate=0.01126343,
col_sample=0.72684326, row_sample=0.86429443,
dropout=0.63078613, n_clusters=2,
type_clust="gmm",
verbose=1, seed = 123,
method="SAMME.R")
start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")
start = time()
print(fit_obj.score(X_test, y_test))
print(f"Elapsed {time() - start}")
preds = fit_obj.predict(X_test)
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
print(metrics.classification_report(preds, y_test))
156 def __init__( 157 self, 158 obj, 159 n_estimators=10, 160 learning_rate=0.1, 161 n_hidden_features=1, 162 reg_lambda=0, 163 reg_alpha=0.5, 164 activation_name="relu", 165 a=0.01, 166 nodes_sim="sobol", 167 bias=True, 168 dropout=0, 169 direct_link=False, 170 n_clusters=2, 171 cluster_encode=True, 172 type_clust="kmeans", 173 type_scaling=("std", "std", "std"), 174 col_sample=1, 175 row_sample=1, 176 seed=123, 177 verbose=1, 178 method="SAMME", 179 backend="cpu", 180 ): 181 super().__init__( 182 obj=obj, 183 n_estimators=n_estimators, 184 learning_rate=learning_rate, 185 n_hidden_features=n_hidden_features, 186 activation_name=activation_name, 187 a=a, 188 nodes_sim=nodes_sim, 189 bias=bias, 190 dropout=dropout, 191 direct_link=direct_link, 192 n_clusters=n_clusters, 193 cluster_encode=cluster_encode, 194 type_clust=type_clust, 195 type_scaling=type_scaling, 196 col_sample=col_sample, 197 row_sample=row_sample, 198 seed=seed, 199 backend=backend, 200 ) 201 202 self.type_fit = "classification" 203 self.verbose = verbose 204 self.method = method 205 self.reg_lambda = reg_lambda 206 self.reg_alpha = reg_alpha 207 208 self.alpha_ = [] 209 self.base_learners_ = dict.fromkeys(range(n_estimators))
211 def fit(self, X, y, sample_weight=None, **kwargs): 212 """Fit Boosting model to training data (X, y). 213 214 Parameters: 215 216 X: {array-like}, shape = [n_samples, n_features] 217 Training vectors, where n_samples is the number 218 of samples and n_features is the number of features. 219 220 y: array-like, shape = [n_samples] 221 Target values. 222 223 **kwargs: additional parameters to be passed to 224 self.cook_training_set or self.obj.fit 225 226 Returns: 227 228 self: object 229 """ 230 231 assert mx.is_factor(y), "y must contain only integers" 232 233 assert self.method in ( 234 "SAMME", 235 "SAMME.R", 236 ), "`method` must be either 'SAMME' or 'SAMME.R'" 237 238 assert (self.reg_lambda <= 1) & ( 239 self.reg_lambda >= 0 240 ), "must have self.reg_lambda <= 1 & self.reg_lambda >= 0" 241 242 assert (self.reg_alpha <= 1) & ( 243 self.reg_alpha >= 0 244 ), "must have self.reg_alpha <= 1 & self.reg_alpha >= 0" 245 246 # training 247 n, p = X.shape 248 self.n_classes = len(np.unique(y)) 249 250 if sample_weight is None: 251 w_m = np.repeat(1.0 / n, n) 252 253 else: 254 w_m = np.asarray(sample_weight) 255 256 base_learner = CustomClassifier( 257 self.obj, 258 n_hidden_features=self.n_hidden_features, 259 activation_name=self.activation_name, 260 a=self.a, 261 nodes_sim=self.nodes_sim, 262 bias=self.bias, 263 dropout=self.dropout, 264 direct_link=self.direct_link, 265 n_clusters=self.n_clusters, 266 type_clust=self.type_clust, 267 type_scaling=self.type_scaling, 268 col_sample=self.col_sample, 269 row_sample=self.row_sample, 270 seed=self.seed, 271 ) 272 273 if self.verbose == 1: 274 pbar = Progbar(self.n_estimators) 275 276 if self.method == "SAMME": 277 err_m = 1e6 278 err_bound = 1 - 1 / self.n_classes 279 self.alpha_.append(1.0) 280 x_range_n = range(n) 281 282 for m in range(self.n_estimators): 283 preds = base_learner.fit( 284 X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs 285 ).predict(X) 286 287 self.base_learners_.update( 288 {m: pickle.loads(pickle.dumps(base_learner, -1))} 289 ) 290 291 cond = [y[i] != preds[i] for i in x_range_n] 292 293 err_m = max( 294 sum([elt[0] * elt[1] for elt in zip(cond, w_m)]), 295 2.220446049250313e-16, 296 ) # sum(w_m) == 1 297 298 if self.reg_lambda > 0: 299 err_m += self.reg_lambda * ( 300 (1 - self.reg_alpha) * 0.5 * sum([x**2 for x in w_m]) 301 + self.reg_alpha * sum([abs(x) for x in w_m]) 302 ) 303 304 err_m = min(err_m, err_bound) 305 306 alpha_m = self.learning_rate * log( 307 (self.n_classes - 1) * (1 - err_m) / err_m 308 ) 309 310 self.alpha_.append(alpha_m) 311 312 w_m_temp = [exp(alpha_m * cond[i]) for i in x_range_n] 313 314 sum_w_m = sum(w_m_temp) 315 316 w_m = np.asarray([w_m_temp[i] / sum_w_m for i in x_range_n]) 317 318 base_learner.set_params(seed=self.seed + (m + 1) * 1000) 319 320 if self.verbose == 1: 321 pbar.update(m) 322 323 if self.verbose == 1: 324 pbar.update(self.n_estimators) 325 326 self.n_estimators = len(self.base_learners_) 327 328 return self 329 330 if self.method == "SAMME.R": 331 Y = mo.one_hot_encode2(y, self.n_classes) 332 333 if sample_weight is None: 334 w_m = np.repeat(1.0 / n, n) # (N, 1) 335 336 else: 337 w_m = np.asarray(sample_weight) 338 339 for m in range(self.n_estimators): 340 probs = base_learner.fit( 341 X, y, sample_weight=np.ravel(w_m, order="C"), **kwargs 342 ).predict_proba(X) 343 344 np.clip( 345 a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs 346 ) 347 348 self.base_learners_.update( 349 {m: pickle.loads(pickle.dumps(base_learner, -1))} 350 ) 351 352 w_m *= np.exp( 353 -1.0 354 * self.learning_rate 355 * (1.0 - 1.0 / self.n_classes) 356 * xlogy(Y, probs).sum(axis=1) 357 ) 358 359 w_m /= np.sum(w_m) 360 361 base_learner.set_params(seed=self.seed + (m + 1) * 1000) 362 363 if self.verbose == 1: 364 pbar.update(m) 365 366 if self.verbose == 1: 367 pbar.update(self.n_estimators) 368 369 self.n_estimators = len(self.base_learners_) 370 371 return self
Fit Boosting model to training data (X, y).
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
373 def predict(self, X, **kwargs): 374 """Predict test data X. 375 376 Parameters: 377 378 X: {array-like}, shape = [n_samples, n_features] 379 Training vectors, where n_samples is the number 380 of samples and n_features is the number of features. 381 382 **kwargs: additional parameters to be passed to 383 self.cook_test_set 384 385 Returns: 386 387 model predictions: {array-like} 388 """ 389 390 return self.predict_proba(X, **kwargs).argmax(axis=1)
Predict test data X.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
392 def predict_proba(self, X, **kwargs): 393 """Predict probabilities for test data X. 394 395 Parameters: 396 397 X: {array-like}, shape = [n_samples, n_features] 398 Training vectors, where n_samples is the number 399 of samples and n_features is the number of features. 400 401 **kwargs: additional parameters to be passed to 402 self.cook_test_set 403 404 Returns: 405 406 probability estimates for test data: {array-like} 407 408 """ 409 410 n_iter = len(self.base_learners_) 411 412 if self.method == "SAMME": 413 ensemble_learner = np.zeros((X.shape[0], self.n_classes)) 414 415 # if self.verbose == 1: 416 # pbar = Progbar(n_iter) 417 418 for idx, base_learner in self.base_learners_.items(): 419 preds = base_learner.predict(X, **kwargs) 420 421 ensemble_learner += self.alpha_[idx] * mo.one_hot_encode2( 422 preds, self.n_classes 423 ) 424 425 # if self.verbose == 1: 426 # pbar.update(idx) 427 428 # if self.verbose == 1: 429 # pbar.update(n_iter) 430 431 expit_ensemble_learner = expit(ensemble_learner) 432 433 sum_ensemble = expit_ensemble_learner.sum(axis=1) 434 435 return expit_ensemble_learner / sum_ensemble[:, None] 436 437 # if self.method == "SAMME.R": 438 ensemble_learner = 0 439 440 # if self.verbose == 1: 441 # pbar = Progbar(n_iter) 442 443 for idx, base_learner in self.base_learners_.items(): 444 probs = base_learner.predict_proba(X, **kwargs) 445 446 np.clip(a=probs, a_min=2.220446049250313e-16, a_max=1.0, out=probs) 447 448 log_preds_proba = np.log(probs) 449 450 ensemble_learner += ( 451 log_preds_proba - log_preds_proba.mean(axis=1)[:, None] 452 ) 453 454 # if self.verbose == 1: 455 # pbar.update(idx) 456 457 ensemble_learner *= self.n_classes - 1 458 459 # if self.verbose == 1: 460 # pbar.update(n_iter) 461 462 expit_ensemble_learner = expit(ensemble_learner) 463 464 sum_ensemble = expit_ensemble_learner.sum(axis=1) 465 466 return expit_ensemble_learner / sum_ensemble[:, None]
Predict probabilities for test data X.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
probability estimates for test data: {array-like}
468 def score(self, X, y, scoring=None, **kwargs): 469 """Score the model on test set features X and response y. 470 471 Parameters: 472 473 X: {array-like}, shape = [n_samples, n_features] 474 Training vectors, where n_samples is the number 475 of samples and n_features is the number of features 476 477 y: array-like, shape = [n_samples] 478 Target values 479 480 scoring: str 481 must be in ('accuracy', 'average_precision', 482 'brier_score_loss', 'f1', 'f1_micro', 483 'f1_macro', 'f1_weighted', 'f1_samples', 484 'neg_log_loss', 'precision', 'recall', 485 'roc_auc') 486 487 **kwargs: additional parameters to be passed to scoring functions 488 489 Returns: 490 491 model scores: {array-like} 492 493 """ 494 495 preds = self.predict(X) 496 497 if scoring is None: 498 scoring = "accuracy" 499 500 # check inputs 501 assert scoring in ( 502 "accuracy", 503 "average_precision", 504 "brier_score_loss", 505 "f1", 506 "f1_micro", 507 "f1_macro", 508 "f1_weighted", 509 "f1_samples", 510 "neg_log_loss", 511 "precision", 512 "recall", 513 "roc_auc", 514 ), "'scoring' should be in ('accuracy', 'average_precision', \ 515 'brier_score_loss', 'f1', 'f1_micro', \ 516 'f1_macro', 'f1_weighted', 'f1_samples', \ 517 'neg_log_loss', 'precision', 'recall', \ 518 'roc_auc')" 519 520 scoring_options = { 521 "accuracy": skm2.accuracy_score, 522 "average_precision": skm2.average_precision_score, 523 "brier_score_loss": skm2.brier_score_loss, 524 "f1": skm2.f1_score, 525 "f1_micro": skm2.f1_score, 526 "f1_macro": skm2.f1_score, 527 "f1_weighted": skm2.f1_score, 528 "f1_samples": skm2.f1_score, 529 "neg_log_loss": skm2.log_loss, 530 "precision": skm2.precision_score, 531 "recall": skm2.recall_score, 532 "roc_auc": skm2.roc_auc_score, 533 } 534 535 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('accuracy', 'average_precision',
'brier_score_loss', 'f1', 'f1_micro',
'f1_macro', 'f1_weighted', 'f1_samples',
'neg_log_loss', 'precision', 'recall',
'roc_auc')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.boosting.bst.Boosting
- obj
- n_estimators
- learning_rate
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
31class Base(BaseEstimator): 32 """Base model from which all the other classes inherit. 33 34 This class contains the most important data preprocessing/feature engineering methods. 35 36 Parameters: 37 38 n_hidden_features: int 39 number of nodes in the hidden layer 40 41 activation_name: str 42 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 43 44 a: float 45 hyperparameter for 'prelu' or 'elu' activation function 46 47 nodes_sim: str 48 type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton', 49 'uniform' 50 51 bias: boolean 52 indicates if the hidden layer contains a bias term (True) or 53 not (False) 54 55 dropout: float 56 regularization parameter; (random) percentage of nodes dropped out 57 of the training 58 59 direct_link: boolean 60 indicates if the original features are included (True) in model's 61 fitting or not (False) 62 63 n_clusters: int 64 number of clusters for type_clust='kmeans' or type_clust='gmm' 65 clustering (could be 0: no clustering) 66 67 cluster_encode: bool 68 defines how the variable containing clusters is treated (default is one-hot); 69 if `False`, then labels are used, without one-hot encoding 70 71 type_clust: str 72 type of clustering method: currently k-means ('kmeans') or Gaussian 73 Mixture Model ('gmm') 74 75 type_scaling: a tuple of 3 strings 76 scaling methods for inputs, hidden layer, and clustering respectively 77 (and when relevant). 78 Currently available: standardization ('std') or MinMax scaling ('minmax') or robust scaling ('robust') 79 80 col_sample: float 81 percentage of features randomly chosen for training 82 83 row_sample: float 84 percentage of rows chosen for training, by stratified bootstrapping 85 86 seed: int 87 reproducibility seed for nodes_sim=='uniform', clustering and dropout 88 89 backend: str 90 "cpu" or "gpu" or "tpu" 91 92 """ 93 94 # construct the object ----- 95 96 def __init__( 97 self, 98 n_hidden_features=5, 99 activation_name="relu", 100 a=0.01, 101 nodes_sim="sobol", 102 bias=True, 103 dropout=0, 104 direct_link=True, 105 n_clusters=2, 106 cluster_encode=True, 107 type_clust="kmeans", 108 type_scaling=("std", "std", "std"), 109 col_sample=1, 110 row_sample=1, 111 seed=123, 112 backend="cpu", 113 ): 114 # input checks ----- 115 116 sys_platform = platform.system() 117 118 if (sys_platform == "Windows") and (backend in ("gpu", "tpu")): 119 warnings.warn( 120 "No GPU/TPU computing on Windows yet, backend set to 'cpu'" 121 ) 122 backend = "cpu" 123 124 assert activation_name in ( 125 "relu", 126 "tanh", 127 "sigmoid", 128 "prelu", 129 "elu", 130 ), "'activation_name' must be in ('relu', 'tanh', 'sigmoid','prelu', 'elu')" 131 132 assert nodes_sim in ( 133 "sobol", 134 "hammersley", 135 "uniform", 136 "halton", 137 ), "'nodes_sim' must be in ('sobol', 'hammersley', 'uniform', 'halton')" 138 139 assert type_clust in ( 140 "kmeans", 141 "gmm", 142 ), "'type_clust' must be in ('kmeans', 'gmm')" 143 144 assert (len(type_scaling) == 3) & all( 145 type_scaling[i] in ("minmax", "std", "robust") 146 for i in range(len(type_scaling)) 147 ), "'type_scaling' must have length 3, and available scaling methods are 'minmax' scaling, standardization ('std') and robust scaling ('robust')" 148 149 assert (col_sample >= 0) & ( 150 col_sample <= 1 151 ), "'col_sample' must be comprised between 0 and 1 (both included)" 152 153 assert backend in ( 154 "cpu", 155 "gpu", 156 "tpu", 157 ), "must have 'backend' in ('cpu', 'gpu', 'tpu')" 158 159 self.n_hidden_features = n_hidden_features 160 self.activation_name = activation_name 161 self.a = a 162 self.nodes_sim = nodes_sim 163 self.bias = bias 164 self.seed = seed 165 self.backend = backend 166 self.dropout = dropout 167 self.direct_link = direct_link 168 self.cluster_encode = cluster_encode 169 self.type_clust = type_clust 170 self.type_scaling = type_scaling 171 self.col_sample = col_sample 172 self.row_sample = row_sample 173 self.n_clusters = n_clusters 174 self.subsampler_ = None 175 self.index_col_ = None 176 self.index_row_ = True 177 self.clustering_obj_ = None 178 self.clustering_scaler_ = None 179 self.nn_scaler_ = None 180 self.scaler_ = None 181 self.encoder_ = None 182 self.W_ = None 183 self.X_ = None 184 self.y_ = None 185 self.y_mean_ = None 186 self.beta_ = None 187 188 # activation function ----- 189 if sys_platform in ("Linux", "Darwin"): 190 activation_options = { 191 "relu": ac.relu if (self.backend == "cpu") else jnn.relu, 192 "tanh": np.tanh if (self.backend == "cpu") else jnp.tanh, 193 "sigmoid": ( 194 ac.sigmoid if (self.backend == "cpu") else jnn.sigmoid 195 ), 196 "prelu": partial(ac.prelu, a=a), 197 "elu": ( 198 partial(ac.elu, a=a) 199 if (self.backend == "cpu") 200 else partial(jnn.elu, a=a) 201 ), 202 } 203 else: # on Windows currently, no JAX 204 activation_options = { 205 "relu": ( 206 ac.relu if (self.backend == "cpu") else NotImplementedError 207 ), 208 "tanh": ( 209 np.tanh if (self.backend == "cpu") else NotImplementedError 210 ), 211 "sigmoid": ( 212 ac.sigmoid 213 if (self.backend == "cpu") 214 else NotImplementedError 215 ), 216 "prelu": partial(ac.prelu, a=a), 217 "elu": ( 218 partial(ac.elu, a=a) 219 if (self.backend == "cpu") 220 else NotImplementedError 221 ), 222 } 223 self.activation_func = activation_options[activation_name] 224 225 # "preprocessing" methods to be inherited ----- 226 227 def encode_clusters(self, X=None, predict=False, **kwargs): # 228 """Create new covariates with kmeans or GMM clustering 229 230 Parameters: 231 232 X: {array-like}, shape = [n_samples, n_features] 233 Training vectors, where n_samples is the number 234 of samples and n_features is the number of features. 235 236 predict: boolean 237 is False on training set and True on test set 238 239 **kwargs: 240 additional parameters to be passed to the 241 clustering method 242 243 Returns: 244 245 Clusters' matrix, one-hot encoded: {array-like} 246 247 """ 248 249 np.random.seed(self.seed) 250 251 if X is None: 252 X = self.X_ 253 254 if isinstance(X, pd.DataFrame): 255 X = copy.deepcopy(X.values.astype(float)) 256 257 if predict is False: # encode training set 258 # scale input data before clustering 259 self.clustering_scaler_, scaled_X = mo.scale_covariates( 260 X, choice=self.type_scaling[2] 261 ) 262 263 self.clustering_obj_, X_clustered = mo.cluster_covariates( 264 scaled_X, 265 self.n_clusters, 266 self.seed, 267 type_clust=self.type_clust, 268 **kwargs 269 ) 270 271 if self.cluster_encode == True: 272 return mo.one_hot_encode(X_clustered, self.n_clusters).astype( 273 np.float16 274 ) 275 276 return X_clustered.astype(np.float16) 277 278 # if predict == True, encode test set 279 X_clustered = self.clustering_obj_.predict( 280 self.clustering_scaler_.transform(X) 281 ) 282 283 if self.cluster_encode == True: 284 return mo.one_hot_encode(X_clustered, self.n_clusters).astype( 285 np.float16 286 ) 287 288 return X_clustered.astype(np.float16) 289 290 def create_layer(self, scaled_X, W=None): 291 """Create hidden layer. 292 293 Parameters: 294 295 scaled_X: {array-like}, shape = [n_samples, n_features] 296 Training vectors, where n_samples is the number 297 of samples and n_features is the number of features 298 299 W: {array-like}, shape = [n_features, hidden_features] 300 if provided, constructs the hidden layer with W; otherwise computed internally 301 302 Returns: 303 304 Hidden layer matrix: {array-like} 305 306 """ 307 308 n_features = scaled_X.shape[1] 309 310 # hash_sim = { 311 # "sobol": generate_sobol, 312 # "hammersley": generate_hammersley, 313 # "uniform": generate_uniform, 314 # "halton": generate_halton 315 # } 316 317 if self.bias is False: # no bias term in the hidden layer 318 if W is None: 319 if self.nodes_sim == "sobol": 320 self.W_ = generate_sobol( 321 n_dims=n_features, 322 n_points=self.n_hidden_features, 323 seed=self.seed, 324 ) 325 elif self.nodes_sim == "hammersley": 326 self.W_ = generate_hammersley( 327 n_dims=n_features, 328 n_points=self.n_hidden_features, 329 seed=self.seed, 330 ) 331 elif self.nodes_sim == "uniform": 332 self.W_ = generate_uniform( 333 n_dims=n_features, 334 n_points=self.n_hidden_features, 335 seed=self.seed, 336 ) 337 else: 338 self.W_ = generate_halton( 339 n_dims=n_features, 340 n_points=self.n_hidden_features, 341 seed=self.seed, 342 ) 343 344 # self.W_ = hash_sim[self.nodes_sim]( 345 # n_dims=n_features, 346 # n_points=self.n_hidden_features, 347 # seed=self.seed, 348 # ) 349 350 assert ( 351 scaled_X.shape[1] == self.W_.shape[0] 352 ), "check dimensions of covariates X and matrix W" 353 354 return mo.dropout( 355 x=self.activation_func( 356 mo.safe_sparse_dot( 357 a=scaled_X, b=self.W_, backend=self.backend 358 ) 359 ), 360 drop_prob=self.dropout, 361 seed=self.seed, 362 ) 363 364 # W is not none 365 assert ( 366 scaled_X.shape[1] == W.shape[0] 367 ), "check dimensions of covariates X and matrix W" 368 369 # self.W_ = W 370 return mo.dropout( 371 x=self.activation_func( 372 mo.safe_sparse_dot(a=scaled_X, b=W, backend=self.backend) 373 ), 374 drop_prob=self.dropout, 375 seed=self.seed, 376 ) 377 378 # with bias term in the hidden layer 379 if W is None: 380 n_features_1 = n_features + 1 381 382 if self.nodes_sim == "sobol": 383 self.W_ = generate_sobol( 384 n_dims=n_features_1, 385 n_points=self.n_hidden_features, 386 seed=self.seed, 387 ) 388 elif self.nodes_sim == "hammersley": 389 self.W_ = generate_hammersley( 390 n_dims=n_features_1, 391 n_points=self.n_hidden_features, 392 seed=self.seed, 393 ) 394 elif self.nodes_sim == "uniform": 395 self.W_ = generate_uniform( 396 n_dims=n_features_1, 397 n_points=self.n_hidden_features, 398 seed=self.seed, 399 ) 400 else: 401 self.W_ = generate_halton( 402 n_dims=n_features_1, 403 n_points=self.n_hidden_features, 404 seed=self.seed, 405 ) 406 407 # self.W_ = hash_sim[self.nodes_sim]( 408 # n_dims=n_features_1, 409 # n_points=self.n_hidden_features, 410 # seed=self.seed, 411 # ) 412 413 return mo.dropout( 414 x=self.activation_func( 415 mo.safe_sparse_dot( 416 a=mo.cbind( 417 np.ones(scaled_X.shape[0]), 418 scaled_X, 419 backend=self.backend, 420 ), 421 b=self.W_, 422 backend=self.backend, 423 ) 424 ), 425 drop_prob=self.dropout, 426 seed=self.seed, 427 ) 428 429 # W is not None 430 # self.W_ = W 431 return mo.dropout( 432 x=self.activation_func( 433 mo.safe_sparse_dot( 434 a=mo.cbind( 435 np.ones(scaled_X.shape[0]), 436 scaled_X, 437 backend=self.backend, 438 ), 439 b=W, 440 backend=self.backend, 441 ) 442 ), 443 drop_prob=self.dropout, 444 seed=self.seed, 445 ) 446 447 def cook_training_set(self, y=None, X=None, W=None, **kwargs): 448 """Create new hidden features for training set, with hidden layer, center the response. 449 450 Parameters: 451 452 y: array-like, shape = [n_samples] 453 Target values 454 455 X: {array-like}, shape = [n_samples, n_features] 456 Training vectors, where n_samples is the number 457 of samples and n_features is the number of features 458 459 W: {array-like}, shape = [n_features, hidden_features] 460 if provided, constructs the hidden layer via W 461 462 Returns: 463 464 (centered response, direct link + hidden layer matrix): {tuple} 465 466 """ 467 468 # either X and y are stored or not 469 # assert ((y is None) & (X is None)) | ((y is not None) & (X is not None)) 470 if self.n_hidden_features > 0: # has a hidden layer 471 assert ( 472 len(self.type_scaling) >= 2 473 ), "must have len(self.type_scaling) >= 2 when self.n_hidden_features > 0" 474 475 if X is None: 476 if self.col_sample == 1: 477 input_X = self.X_ 478 else: 479 n_features = self.X_.shape[1] 480 new_n_features = int(np.ceil(n_features * self.col_sample)) 481 assert ( 482 new_n_features >= 1 483 ), "check class attribute 'col_sample' and the number of covariates provided for X" 484 np.random.seed(self.seed) 485 index_col = np.random.choice( 486 range(n_features), size=new_n_features, replace=False 487 ) 488 self.index_col_ = index_col 489 input_X = self.X_[:, self.index_col_] 490 491 else: # X is not None # keep X vs self.X_ 492 if isinstance(X, pd.DataFrame): 493 X = copy.deepcopy(X.values.astype(float)) 494 495 if self.col_sample == 1: 496 input_X = X 497 else: 498 n_features = X.shape[1] 499 new_n_features = int(np.ceil(n_features * self.col_sample)) 500 assert ( 501 new_n_features >= 1 502 ), "check class attribute 'col_sample' and the number of covariates provided for X" 503 np.random.seed(self.seed) 504 index_col = np.random.choice( 505 range(n_features), size=new_n_features, replace=False 506 ) 507 self.index_col_ = index_col 508 input_X = X[:, self.index_col_] 509 510 if ( 511 self.n_clusters <= 0 512 ): # data without any clustering: self.n_clusters is None ----- 513 if self.n_hidden_features > 0: # with hidden layer 514 self.nn_scaler_, scaled_X = mo.scale_covariates( 515 input_X, choice=self.type_scaling[1] 516 ) 517 Phi_X = ( 518 self.create_layer(scaled_X) 519 if W is None 520 else self.create_layer(scaled_X, W=W) 521 ) 522 Z = ( 523 mo.cbind(input_X, Phi_X, backend=self.backend) 524 if self.direct_link is True 525 else Phi_X 526 ) 527 self.scaler_, scaled_Z = mo.scale_covariates( 528 Z, choice=self.type_scaling[0] 529 ) 530 else: # no hidden layer 531 Z = input_X 532 self.scaler_, scaled_Z = mo.scale_covariates( 533 Z, choice=self.type_scaling[0] 534 ) 535 else: # data with clustering: self.n_clusters is not None ----- # keep 536 augmented_X = mo.cbind( 537 input_X, 538 self.encode_clusters(input_X, **kwargs), 539 backend=self.backend, 540 ) 541 542 if self.n_hidden_features > 0: # with hidden layer 543 self.nn_scaler_, scaled_X = mo.scale_covariates( 544 augmented_X, choice=self.type_scaling[1] 545 ) 546 Phi_X = ( 547 self.create_layer(scaled_X) 548 if W is None 549 else self.create_layer(scaled_X, W=W) 550 ) 551 Z = ( 552 mo.cbind(augmented_X, Phi_X, backend=self.backend) 553 if self.direct_link is True 554 else Phi_X 555 ) 556 self.scaler_, scaled_Z = mo.scale_covariates( 557 Z, choice=self.type_scaling[0] 558 ) 559 else: # no hidden layer 560 Z = augmented_X 561 self.scaler_, scaled_Z = mo.scale_covariates( 562 Z, choice=self.type_scaling[0] 563 ) 564 565 # Returning model inputs ----- 566 if mx.is_factor(y) is False: # regression 567 # center y 568 if y is None: 569 self.y_mean_, centered_y = mo.center_response(self.y_) 570 else: 571 self.y_mean_, centered_y = mo.center_response(y) 572 573 # y is subsampled 574 if self.row_sample < 1: 575 n, p = Z.shape 576 577 self.subsampler_ = ( 578 SubSampler( 579 y=self.y_, row_sample=self.row_sample, seed=self.seed 580 ) 581 if y is None 582 else SubSampler( 583 y=y, row_sample=self.row_sample, seed=self.seed 584 ) 585 ) 586 587 self.index_row_ = self.subsampler_.subsample() 588 589 n_row_sample = len(self.index_row_) 590 # regression 591 return ( 592 centered_y[self.index_row_].reshape(n_row_sample), 593 self.scaler_.transform( 594 Z[self.index_row_, :].reshape(n_row_sample, p) 595 ), 596 ) 597 # y is not subsampled 598 # regression 599 return (centered_y, self.scaler_.transform(Z)) 600 601 # classification 602 # y is subsampled 603 if self.row_sample < 1: 604 n, p = Z.shape 605 606 self.subsampler_ = ( 607 SubSampler( 608 y=self.y_, row_sample=self.row_sample, seed=self.seed 609 ) 610 if y is None 611 else SubSampler(y=y, row_sample=self.row_sample, seed=self.seed) 612 ) 613 614 self.index_row_ = self.subsampler_.subsample() 615 616 n_row_sample = len(self.index_row_) 617 # classification 618 return ( 619 y[self.index_row_].reshape(n_row_sample), 620 self.scaler_.transform( 621 Z[self.index_row_, :].reshape(n_row_sample, p) 622 ), 623 ) 624 # y is not subsampled 625 # classification 626 return (y, self.scaler_.transform(Z)) 627 628 def cook_test_set(self, X, **kwargs): 629 """Transform data from test set, with hidden layer. 630 631 Parameters: 632 633 X: {array-like}, shape = [n_samples, n_features] 634 Training vectors, where n_samples is the number 635 of samples and n_features is the number of features 636 637 **kwargs: additional parameters to be passed to self.encode_cluster 638 639 Returns: 640 641 Transformed test set : {array-like} 642 """ 643 644 if isinstance(X, pd.DataFrame): 645 X = copy.deepcopy(X.values.astype(float)) 646 647 if ( 648 self.n_clusters == 0 649 ): # data without clustering: self.n_clusters is None ----- 650 if self.n_hidden_features > 0: 651 # if hidden layer 652 scaled_X = ( 653 self.nn_scaler_.transform(X) 654 if (self.col_sample == 1) 655 else self.nn_scaler_.transform(X[:, self.index_col_]) 656 ) 657 Phi_X = self.create_layer(scaled_X, self.W_) 658 if self.direct_link == True: 659 return self.scaler_.transform( 660 mo.cbind(scaled_X, Phi_X, backend=self.backend) 661 ) 662 # when self.direct_link == False 663 return self.scaler_.transform(Phi_X) 664 # if no hidden layer # self.n_hidden_features == 0 665 return self.scaler_.transform(X) 666 667 # data with clustering: self.n_clusters > 0 ----- 668 if self.col_sample == 1: 669 predicted_clusters = self.encode_clusters( 670 X=X, predict=True, **kwargs 671 ) 672 augmented_X = mo.cbind(X, predicted_clusters, backend=self.backend) 673 else: 674 predicted_clusters = self.encode_clusters( 675 X=X[:, self.index_col_], predict=True, **kwargs 676 ) 677 augmented_X = mo.cbind( 678 X[:, self.index_col_], predicted_clusters, backend=self.backend 679 ) 680 681 if self.n_hidden_features > 0: # if hidden layer 682 scaled_X = self.nn_scaler_.transform(augmented_X) 683 Phi_X = self.create_layer(scaled_X, self.W_) 684 if self.direct_link == True: 685 return self.scaler_.transform( 686 mo.cbind(augmented_X, Phi_X, backend=self.backend) 687 ) 688 return self.scaler_.transform(Phi_X) 689 690 # if no hidden layer 691 return self.scaler_.transform(augmented_X)
Base model from which all the other classes inherit.
This class contains the most important data preprocessing/feature engineering methods.
Parameters:
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or
not (False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original features are included (True) in model's
fitting or not (False)
n_clusters: int
number of clusters for type_clust='kmeans' or type_clust='gmm'
clustering (could be 0: no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot);
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax') or robust scaling ('robust')
col_sample: float
percentage of features randomly chosen for training
row_sample: float
percentage of rows chosen for training, by stratified bootstrapping
seed: int
reproducibility seed for nodes_sim=='uniform', clustering and dropout
backend: str
"cpu" or "gpu" or "tpu"
96 def __init__( 97 self, 98 n_hidden_features=5, 99 activation_name="relu", 100 a=0.01, 101 nodes_sim="sobol", 102 bias=True, 103 dropout=0, 104 direct_link=True, 105 n_clusters=2, 106 cluster_encode=True, 107 type_clust="kmeans", 108 type_scaling=("std", "std", "std"), 109 col_sample=1, 110 row_sample=1, 111 seed=123, 112 backend="cpu", 113 ): 114 # input checks ----- 115 116 sys_platform = platform.system() 117 118 if (sys_platform == "Windows") and (backend in ("gpu", "tpu")): 119 warnings.warn( 120 "No GPU/TPU computing on Windows yet, backend set to 'cpu'" 121 ) 122 backend = "cpu" 123 124 assert activation_name in ( 125 "relu", 126 "tanh", 127 "sigmoid", 128 "prelu", 129 "elu", 130 ), "'activation_name' must be in ('relu', 'tanh', 'sigmoid','prelu', 'elu')" 131 132 assert nodes_sim in ( 133 "sobol", 134 "hammersley", 135 "uniform", 136 "halton", 137 ), "'nodes_sim' must be in ('sobol', 'hammersley', 'uniform', 'halton')" 138 139 assert type_clust in ( 140 "kmeans", 141 "gmm", 142 ), "'type_clust' must be in ('kmeans', 'gmm')" 143 144 assert (len(type_scaling) == 3) & all( 145 type_scaling[i] in ("minmax", "std", "robust") 146 for i in range(len(type_scaling)) 147 ), "'type_scaling' must have length 3, and available scaling methods are 'minmax' scaling, standardization ('std') and robust scaling ('robust')" 148 149 assert (col_sample >= 0) & ( 150 col_sample <= 1 151 ), "'col_sample' must be comprised between 0 and 1 (both included)" 152 153 assert backend in ( 154 "cpu", 155 "gpu", 156 "tpu", 157 ), "must have 'backend' in ('cpu', 'gpu', 'tpu')" 158 159 self.n_hidden_features = n_hidden_features 160 self.activation_name = activation_name 161 self.a = a 162 self.nodes_sim = nodes_sim 163 self.bias = bias 164 self.seed = seed 165 self.backend = backend 166 self.dropout = dropout 167 self.direct_link = direct_link 168 self.cluster_encode = cluster_encode 169 self.type_clust = type_clust 170 self.type_scaling = type_scaling 171 self.col_sample = col_sample 172 self.row_sample = row_sample 173 self.n_clusters = n_clusters 174 self.subsampler_ = None 175 self.index_col_ = None 176 self.index_row_ = True 177 self.clustering_obj_ = None 178 self.clustering_scaler_ = None 179 self.nn_scaler_ = None 180 self.scaler_ = None 181 self.encoder_ = None 182 self.W_ = None 183 self.X_ = None 184 self.y_ = None 185 self.y_mean_ = None 186 self.beta_ = None 187 188 # activation function ----- 189 if sys_platform in ("Linux", "Darwin"): 190 activation_options = { 191 "relu": ac.relu if (self.backend == "cpu") else jnn.relu, 192 "tanh": np.tanh if (self.backend == "cpu") else jnp.tanh, 193 "sigmoid": ( 194 ac.sigmoid if (self.backend == "cpu") else jnn.sigmoid 195 ), 196 "prelu": partial(ac.prelu, a=a), 197 "elu": ( 198 partial(ac.elu, a=a) 199 if (self.backend == "cpu") 200 else partial(jnn.elu, a=a) 201 ), 202 } 203 else: # on Windows currently, no JAX 204 activation_options = { 205 "relu": ( 206 ac.relu if (self.backend == "cpu") else NotImplementedError 207 ), 208 "tanh": ( 209 np.tanh if (self.backend == "cpu") else NotImplementedError 210 ), 211 "sigmoid": ( 212 ac.sigmoid 213 if (self.backend == "cpu") 214 else NotImplementedError 215 ), 216 "prelu": partial(ac.prelu, a=a), 217 "elu": ( 218 partial(ac.elu, a=a) 219 if (self.backend == "cpu") 220 else NotImplementedError 221 ), 222 } 223 self.activation_func = activation_options[activation_name]
227 def encode_clusters(self, X=None, predict=False, **kwargs): # 228 """Create new covariates with kmeans or GMM clustering 229 230 Parameters: 231 232 X: {array-like}, shape = [n_samples, n_features] 233 Training vectors, where n_samples is the number 234 of samples and n_features is the number of features. 235 236 predict: boolean 237 is False on training set and True on test set 238 239 **kwargs: 240 additional parameters to be passed to the 241 clustering method 242 243 Returns: 244 245 Clusters' matrix, one-hot encoded: {array-like} 246 247 """ 248 249 np.random.seed(self.seed) 250 251 if X is None: 252 X = self.X_ 253 254 if isinstance(X, pd.DataFrame): 255 X = copy.deepcopy(X.values.astype(float)) 256 257 if predict is False: # encode training set 258 # scale input data before clustering 259 self.clustering_scaler_, scaled_X = mo.scale_covariates( 260 X, choice=self.type_scaling[2] 261 ) 262 263 self.clustering_obj_, X_clustered = mo.cluster_covariates( 264 scaled_X, 265 self.n_clusters, 266 self.seed, 267 type_clust=self.type_clust, 268 **kwargs 269 ) 270 271 if self.cluster_encode == True: 272 return mo.one_hot_encode(X_clustered, self.n_clusters).astype( 273 np.float16 274 ) 275 276 return X_clustered.astype(np.float16) 277 278 # if predict == True, encode test set 279 X_clustered = self.clustering_obj_.predict( 280 self.clustering_scaler_.transform(X) 281 ) 282 283 if self.cluster_encode == True: 284 return mo.one_hot_encode(X_clustered, self.n_clusters).astype( 285 np.float16 286 ) 287 288 return X_clustered.astype(np.float16)
Create new covariates with kmeans or GMM clustering
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
predict: boolean
is False on training set and True on test set
**kwargs:
additional parameters to be passed to the
clustering method
Returns:
Clusters' matrix, one-hot encoded: {array-like}
290 def create_layer(self, scaled_X, W=None): 291 """Create hidden layer. 292 293 Parameters: 294 295 scaled_X: {array-like}, shape = [n_samples, n_features] 296 Training vectors, where n_samples is the number 297 of samples and n_features is the number of features 298 299 W: {array-like}, shape = [n_features, hidden_features] 300 if provided, constructs the hidden layer with W; otherwise computed internally 301 302 Returns: 303 304 Hidden layer matrix: {array-like} 305 306 """ 307 308 n_features = scaled_X.shape[1] 309 310 # hash_sim = { 311 # "sobol": generate_sobol, 312 # "hammersley": generate_hammersley, 313 # "uniform": generate_uniform, 314 # "halton": generate_halton 315 # } 316 317 if self.bias is False: # no bias term in the hidden layer 318 if W is None: 319 if self.nodes_sim == "sobol": 320 self.W_ = generate_sobol( 321 n_dims=n_features, 322 n_points=self.n_hidden_features, 323 seed=self.seed, 324 ) 325 elif self.nodes_sim == "hammersley": 326 self.W_ = generate_hammersley( 327 n_dims=n_features, 328 n_points=self.n_hidden_features, 329 seed=self.seed, 330 ) 331 elif self.nodes_sim == "uniform": 332 self.W_ = generate_uniform( 333 n_dims=n_features, 334 n_points=self.n_hidden_features, 335 seed=self.seed, 336 ) 337 else: 338 self.W_ = generate_halton( 339 n_dims=n_features, 340 n_points=self.n_hidden_features, 341 seed=self.seed, 342 ) 343 344 # self.W_ = hash_sim[self.nodes_sim]( 345 # n_dims=n_features, 346 # n_points=self.n_hidden_features, 347 # seed=self.seed, 348 # ) 349 350 assert ( 351 scaled_X.shape[1] == self.W_.shape[0] 352 ), "check dimensions of covariates X and matrix W" 353 354 return mo.dropout( 355 x=self.activation_func( 356 mo.safe_sparse_dot( 357 a=scaled_X, b=self.W_, backend=self.backend 358 ) 359 ), 360 drop_prob=self.dropout, 361 seed=self.seed, 362 ) 363 364 # W is not none 365 assert ( 366 scaled_X.shape[1] == W.shape[0] 367 ), "check dimensions of covariates X and matrix W" 368 369 # self.W_ = W 370 return mo.dropout( 371 x=self.activation_func( 372 mo.safe_sparse_dot(a=scaled_X, b=W, backend=self.backend) 373 ), 374 drop_prob=self.dropout, 375 seed=self.seed, 376 ) 377 378 # with bias term in the hidden layer 379 if W is None: 380 n_features_1 = n_features + 1 381 382 if self.nodes_sim == "sobol": 383 self.W_ = generate_sobol( 384 n_dims=n_features_1, 385 n_points=self.n_hidden_features, 386 seed=self.seed, 387 ) 388 elif self.nodes_sim == "hammersley": 389 self.W_ = generate_hammersley( 390 n_dims=n_features_1, 391 n_points=self.n_hidden_features, 392 seed=self.seed, 393 ) 394 elif self.nodes_sim == "uniform": 395 self.W_ = generate_uniform( 396 n_dims=n_features_1, 397 n_points=self.n_hidden_features, 398 seed=self.seed, 399 ) 400 else: 401 self.W_ = generate_halton( 402 n_dims=n_features_1, 403 n_points=self.n_hidden_features, 404 seed=self.seed, 405 ) 406 407 # self.W_ = hash_sim[self.nodes_sim]( 408 # n_dims=n_features_1, 409 # n_points=self.n_hidden_features, 410 # seed=self.seed, 411 # ) 412 413 return mo.dropout( 414 x=self.activation_func( 415 mo.safe_sparse_dot( 416 a=mo.cbind( 417 np.ones(scaled_X.shape[0]), 418 scaled_X, 419 backend=self.backend, 420 ), 421 b=self.W_, 422 backend=self.backend, 423 ) 424 ), 425 drop_prob=self.dropout, 426 seed=self.seed, 427 ) 428 429 # W is not None 430 # self.W_ = W 431 return mo.dropout( 432 x=self.activation_func( 433 mo.safe_sparse_dot( 434 a=mo.cbind( 435 np.ones(scaled_X.shape[0]), 436 scaled_X, 437 backend=self.backend, 438 ), 439 b=W, 440 backend=self.backend, 441 ) 442 ), 443 drop_prob=self.dropout, 444 seed=self.seed, 445 )
Create hidden layer.
Parameters:
scaled_X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
W: {array-like}, shape = [n_features, hidden_features]
if provided, constructs the hidden layer with W; otherwise computed internally
Returns:
Hidden layer matrix: {array-like}
447 def cook_training_set(self, y=None, X=None, W=None, **kwargs): 448 """Create new hidden features for training set, with hidden layer, center the response. 449 450 Parameters: 451 452 y: array-like, shape = [n_samples] 453 Target values 454 455 X: {array-like}, shape = [n_samples, n_features] 456 Training vectors, where n_samples is the number 457 of samples and n_features is the number of features 458 459 W: {array-like}, shape = [n_features, hidden_features] 460 if provided, constructs the hidden layer via W 461 462 Returns: 463 464 (centered response, direct link + hidden layer matrix): {tuple} 465 466 """ 467 468 # either X and y are stored or not 469 # assert ((y is None) & (X is None)) | ((y is not None) & (X is not None)) 470 if self.n_hidden_features > 0: # has a hidden layer 471 assert ( 472 len(self.type_scaling) >= 2 473 ), "must have len(self.type_scaling) >= 2 when self.n_hidden_features > 0" 474 475 if X is None: 476 if self.col_sample == 1: 477 input_X = self.X_ 478 else: 479 n_features = self.X_.shape[1] 480 new_n_features = int(np.ceil(n_features * self.col_sample)) 481 assert ( 482 new_n_features >= 1 483 ), "check class attribute 'col_sample' and the number of covariates provided for X" 484 np.random.seed(self.seed) 485 index_col = np.random.choice( 486 range(n_features), size=new_n_features, replace=False 487 ) 488 self.index_col_ = index_col 489 input_X = self.X_[:, self.index_col_] 490 491 else: # X is not None # keep X vs self.X_ 492 if isinstance(X, pd.DataFrame): 493 X = copy.deepcopy(X.values.astype(float)) 494 495 if self.col_sample == 1: 496 input_X = X 497 else: 498 n_features = X.shape[1] 499 new_n_features = int(np.ceil(n_features * self.col_sample)) 500 assert ( 501 new_n_features >= 1 502 ), "check class attribute 'col_sample' and the number of covariates provided for X" 503 np.random.seed(self.seed) 504 index_col = np.random.choice( 505 range(n_features), size=new_n_features, replace=False 506 ) 507 self.index_col_ = index_col 508 input_X = X[:, self.index_col_] 509 510 if ( 511 self.n_clusters <= 0 512 ): # data without any clustering: self.n_clusters is None ----- 513 if self.n_hidden_features > 0: # with hidden layer 514 self.nn_scaler_, scaled_X = mo.scale_covariates( 515 input_X, choice=self.type_scaling[1] 516 ) 517 Phi_X = ( 518 self.create_layer(scaled_X) 519 if W is None 520 else self.create_layer(scaled_X, W=W) 521 ) 522 Z = ( 523 mo.cbind(input_X, Phi_X, backend=self.backend) 524 if self.direct_link is True 525 else Phi_X 526 ) 527 self.scaler_, scaled_Z = mo.scale_covariates( 528 Z, choice=self.type_scaling[0] 529 ) 530 else: # no hidden layer 531 Z = input_X 532 self.scaler_, scaled_Z = mo.scale_covariates( 533 Z, choice=self.type_scaling[0] 534 ) 535 else: # data with clustering: self.n_clusters is not None ----- # keep 536 augmented_X = mo.cbind( 537 input_X, 538 self.encode_clusters(input_X, **kwargs), 539 backend=self.backend, 540 ) 541 542 if self.n_hidden_features > 0: # with hidden layer 543 self.nn_scaler_, scaled_X = mo.scale_covariates( 544 augmented_X, choice=self.type_scaling[1] 545 ) 546 Phi_X = ( 547 self.create_layer(scaled_X) 548 if W is None 549 else self.create_layer(scaled_X, W=W) 550 ) 551 Z = ( 552 mo.cbind(augmented_X, Phi_X, backend=self.backend) 553 if self.direct_link is True 554 else Phi_X 555 ) 556 self.scaler_, scaled_Z = mo.scale_covariates( 557 Z, choice=self.type_scaling[0] 558 ) 559 else: # no hidden layer 560 Z = augmented_X 561 self.scaler_, scaled_Z = mo.scale_covariates( 562 Z, choice=self.type_scaling[0] 563 ) 564 565 # Returning model inputs ----- 566 if mx.is_factor(y) is False: # regression 567 # center y 568 if y is None: 569 self.y_mean_, centered_y = mo.center_response(self.y_) 570 else: 571 self.y_mean_, centered_y = mo.center_response(y) 572 573 # y is subsampled 574 if self.row_sample < 1: 575 n, p = Z.shape 576 577 self.subsampler_ = ( 578 SubSampler( 579 y=self.y_, row_sample=self.row_sample, seed=self.seed 580 ) 581 if y is None 582 else SubSampler( 583 y=y, row_sample=self.row_sample, seed=self.seed 584 ) 585 ) 586 587 self.index_row_ = self.subsampler_.subsample() 588 589 n_row_sample = len(self.index_row_) 590 # regression 591 return ( 592 centered_y[self.index_row_].reshape(n_row_sample), 593 self.scaler_.transform( 594 Z[self.index_row_, :].reshape(n_row_sample, p) 595 ), 596 ) 597 # y is not subsampled 598 # regression 599 return (centered_y, self.scaler_.transform(Z)) 600 601 # classification 602 # y is subsampled 603 if self.row_sample < 1: 604 n, p = Z.shape 605 606 self.subsampler_ = ( 607 SubSampler( 608 y=self.y_, row_sample=self.row_sample, seed=self.seed 609 ) 610 if y is None 611 else SubSampler(y=y, row_sample=self.row_sample, seed=self.seed) 612 ) 613 614 self.index_row_ = self.subsampler_.subsample() 615 616 n_row_sample = len(self.index_row_) 617 # classification 618 return ( 619 y[self.index_row_].reshape(n_row_sample), 620 self.scaler_.transform( 621 Z[self.index_row_, :].reshape(n_row_sample, p) 622 ), 623 ) 624 # y is not subsampled 625 # classification 626 return (y, self.scaler_.transform(Z))
Create new hidden features for training set, with hidden layer, center the response.
Parameters:
y: array-like, shape = [n_samples]
Target values
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
W: {array-like}, shape = [n_features, hidden_features]
if provided, constructs the hidden layer via W
Returns:
(centered response, direct link + hidden layer matrix): {tuple}
628 def cook_test_set(self, X, **kwargs): 629 """Transform data from test set, with hidden layer. 630 631 Parameters: 632 633 X: {array-like}, shape = [n_samples, n_features] 634 Training vectors, where n_samples is the number 635 of samples and n_features is the number of features 636 637 **kwargs: additional parameters to be passed to self.encode_cluster 638 639 Returns: 640 641 Transformed test set : {array-like} 642 """ 643 644 if isinstance(X, pd.DataFrame): 645 X = copy.deepcopy(X.values.astype(float)) 646 647 if ( 648 self.n_clusters == 0 649 ): # data without clustering: self.n_clusters is None ----- 650 if self.n_hidden_features > 0: 651 # if hidden layer 652 scaled_X = ( 653 self.nn_scaler_.transform(X) 654 if (self.col_sample == 1) 655 else self.nn_scaler_.transform(X[:, self.index_col_]) 656 ) 657 Phi_X = self.create_layer(scaled_X, self.W_) 658 if self.direct_link == True: 659 return self.scaler_.transform( 660 mo.cbind(scaled_X, Phi_X, backend=self.backend) 661 ) 662 # when self.direct_link == False 663 return self.scaler_.transform(Phi_X) 664 # if no hidden layer # self.n_hidden_features == 0 665 return self.scaler_.transform(X) 666 667 # data with clustering: self.n_clusters > 0 ----- 668 if self.col_sample == 1: 669 predicted_clusters = self.encode_clusters( 670 X=X, predict=True, **kwargs 671 ) 672 augmented_X = mo.cbind(X, predicted_clusters, backend=self.backend) 673 else: 674 predicted_clusters = self.encode_clusters( 675 X=X[:, self.index_col_], predict=True, **kwargs 676 ) 677 augmented_X = mo.cbind( 678 X[:, self.index_col_], predicted_clusters, backend=self.backend 679 ) 680 681 if self.n_hidden_features > 0: # if hidden layer 682 scaled_X = self.nn_scaler_.transform(augmented_X) 683 Phi_X = self.create_layer(scaled_X, self.W_) 684 if self.direct_link == True: 685 return self.scaler_.transform( 686 mo.cbind(augmented_X, Phi_X, backend=self.backend) 687 ) 688 return self.scaler_.transform(Phi_X) 689 690 # if no hidden layer 691 return self.scaler_.transform(augmented_X)
Transform data from test set, with hidden layer.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
**kwargs: additional parameters to be passed to self.encode_cluster
Returns:
Transformed test set : {array-like}
Inherited Members
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
15class BaseRegressor(Base, RegressorMixin): 16 """Random Vector Functional Link Network regression without shrinkage 17 18 Parameters: 19 20 n_hidden_features: int 21 number of nodes in the hidden layer 22 23 activation_name: str 24 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 25 26 a: float 27 hyperparameter for 'prelu' or 'elu' activation function 28 29 nodes_sim: str 30 type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton', 31 'uniform' 32 33 bias: boolean 34 indicates if the hidden layer contains a bias term (True) or 35 not (False) 36 37 dropout: float 38 regularization parameter; (random) percentage of nodes dropped out 39 of the training 40 41 direct_link: boolean 42 indicates if the original features are included (True) in model's 43 fitting or not (False) 44 45 n_clusters: int 46 number of clusters for type_clust='kmeans' or type_clust='gmm' 47 clustering (could be 0: no clustering) 48 49 cluster_encode: bool 50 defines how the variable containing clusters is treated (default is one-hot); 51 if `False`, then labels are used, without one-hot encoding 52 53 type_clust: str 54 type of clustering method: currently k-means ('kmeans') or Gaussian 55 Mixture Model ('gmm') 56 57 type_scaling: a tuple of 3 strings 58 scaling methods for inputs, hidden layer, and clustering respectively 59 (and when relevant). 60 Currently available: standardization ('std') or MinMax scaling ('minmax') 61 62 col_sample: float 63 percentage of features randomly chosen for training 64 65 row_sample: float 66 percentage of rows chosen for training, by stratified bootstrapping 67 68 seed: int 69 reproducibility seed for nodes_sim=='uniform', clustering and dropout 70 71 backend: str 72 "cpu" or "gpu" or "tpu" 73 74 Attributes: 75 76 beta_: vector 77 regression coefficients 78 79 GCV_: float 80 Generalized Cross-Validation error 81 82 """ 83 84 # construct the object ----- 85 86 def __init__( 87 self, 88 n_hidden_features=5, 89 activation_name="relu", 90 a=0.01, 91 nodes_sim="sobol", 92 bias=True, 93 dropout=0, 94 direct_link=True, 95 n_clusters=2, 96 cluster_encode=True, 97 type_clust="kmeans", 98 type_scaling=("std", "std", "std"), 99 col_sample=1, 100 row_sample=1, 101 seed=123, 102 backend="cpu", 103 ): 104 super().__init__( 105 n_hidden_features=n_hidden_features, 106 activation_name=activation_name, 107 a=a, 108 nodes_sim=nodes_sim, 109 bias=bias, 110 dropout=dropout, 111 direct_link=direct_link, 112 n_clusters=n_clusters, 113 cluster_encode=cluster_encode, 114 type_clust=type_clust, 115 type_scaling=type_scaling, 116 col_sample=col_sample, 117 row_sample=row_sample, 118 seed=seed, 119 backend=backend, 120 ) 121 122 def fit(self, X, y, **kwargs): 123 """Fit BaseRegressor to training data (X, y) 124 125 Parameters: 126 127 X: {array-like}, shape = [n_samples, n_features] 128 Training vectors, where n_samples is the number 129 of samples and n_features is the number of features 130 131 y: array-like, shape = [n_samples] 132 Target values 133 134 **kwargs: additional parameters to be passed to self.cook_training_set 135 136 Returns: 137 138 self: object 139 """ 140 141 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 142 143 fit_obj = lmf.beta_Sigma_hat( 144 X=scaled_Z, y=centered_y, backend=self.backend 145 ) 146 147 self.beta_ = fit_obj["beta_hat"] 148 149 self.GCV_ = fit_obj["GCV"] 150 151 return self 152 153 def predict(self, X, **kwargs): 154 """Predict test data X. 155 156 Parameters: 157 158 X: {array-like}, shape = [n_samples, n_features] 159 Training vectors, where n_samples is the number 160 of samples and n_features is the number of features 161 162 **kwargs: additional parameters to be passed to self.cook_test_set 163 164 Returns: 165 166 model predictions: {array-like} 167 """ 168 169 if len(X.shape) == 1: 170 n_features = X.shape[0] 171 new_X = mo.rbind( 172 X.reshape(1, n_features), 173 np.ones(n_features).reshape(1, n_features), 174 ) 175 176 return ( 177 self.y_mean_ 178 + mo.safe_sparse_dot( 179 a=self.cook_test_set(new_X, **kwargs), 180 b=self.beta_, 181 backend=self.backend, 182 ) 183 )[0] 184 185 return self.y_mean_ + mo.safe_sparse_dot( 186 a=self.cook_test_set(X, **kwargs), 187 b=self.beta_, 188 backend=self.backend, 189 ) 190 191 def score(self, X, y, scoring=None, **kwargs): 192 """Score the model on test set features X and response y. 193 194 Parameters: 195 196 X: {array-like}, shape = [n_samples, n_features] 197 Training vectors, where n_samples is the number 198 of samples and n_features is the number of features 199 200 y: array-like, shape = [n_samples] 201 Target values 202 203 scoring: str 204 must be in ('explained_variance', 'neg_mean_absolute_error', 205 'neg_mean_squared_error', 'neg_mean_squared_log_error', 206 'neg_median_absolute_error', 'r2') 207 208 **kwargs: additional parameters to be passed to scoring functions 209 210 Returns: 211 212 model scores: {array-like} 213 214 """ 215 216 preds = self.predict(X) 217 218 if type(preds) == tuple: # if there are std. devs in the predictions 219 preds = preds[0] 220 221 if scoring is None: 222 scoring = "neg_mean_squared_error" 223 224 # check inputs 225 assert scoring in ( 226 "explained_variance", 227 "neg_mean_absolute_error", 228 "neg_mean_squared_error", 229 "neg_mean_squared_log_error", 230 "neg_median_absolute_error", 231 "r2", 232 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 233 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 234 'neg_median_absolute_error', 'r2')" 235 236 scoring_options = { 237 "explained_variance": skm.explained_variance_score, 238 "neg_mean_absolute_error": skm.median_absolute_error, 239 "neg_mean_squared_error": skm.mean_squared_error, 240 "neg_mean_squared_log_error": skm.mean_squared_log_error, 241 "neg_median_absolute_error": skm.median_absolute_error, 242 "r2": skm.r2_score, 243 } 244 245 return scoring_options[scoring](y, preds, **kwargs)
Random Vector Functional Link Network regression without shrinkage
Parameters:
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for hidden layer nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or
not (False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original features are included (True) in model's
fitting or not (False)
n_clusters: int
number of clusters for type_clust='kmeans' or type_clust='gmm'
clustering (could be 0: no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot);
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
col_sample: float
percentage of features randomly chosen for training
row_sample: float
percentage of rows chosen for training, by stratified bootstrapping
seed: int
reproducibility seed for nodes_sim=='uniform', clustering and dropout
backend: str
"cpu" or "gpu" or "tpu"
Attributes:
beta_: vector
regression coefficients
GCV_: float
Generalized Cross-Validation error
86 def __init__( 87 self, 88 n_hidden_features=5, 89 activation_name="relu", 90 a=0.01, 91 nodes_sim="sobol", 92 bias=True, 93 dropout=0, 94 direct_link=True, 95 n_clusters=2, 96 cluster_encode=True, 97 type_clust="kmeans", 98 type_scaling=("std", "std", "std"), 99 col_sample=1, 100 row_sample=1, 101 seed=123, 102 backend="cpu", 103 ): 104 super().__init__( 105 n_hidden_features=n_hidden_features, 106 activation_name=activation_name, 107 a=a, 108 nodes_sim=nodes_sim, 109 bias=bias, 110 dropout=dropout, 111 direct_link=direct_link, 112 n_clusters=n_clusters, 113 cluster_encode=cluster_encode, 114 type_clust=type_clust, 115 type_scaling=type_scaling, 116 col_sample=col_sample, 117 row_sample=row_sample, 118 seed=seed, 119 backend=backend, 120 )
122 def fit(self, X, y, **kwargs): 123 """Fit BaseRegressor to training data (X, y) 124 125 Parameters: 126 127 X: {array-like}, shape = [n_samples, n_features] 128 Training vectors, where n_samples is the number 129 of samples and n_features is the number of features 130 131 y: array-like, shape = [n_samples] 132 Target values 133 134 **kwargs: additional parameters to be passed to self.cook_training_set 135 136 Returns: 137 138 self: object 139 """ 140 141 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 142 143 fit_obj = lmf.beta_Sigma_hat( 144 X=scaled_Z, y=centered_y, backend=self.backend 145 ) 146 147 self.beta_ = fit_obj["beta_hat"] 148 149 self.GCV_ = fit_obj["GCV"] 150 151 return self
Fit BaseRegressor to training data (X, y)
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
**kwargs: additional parameters to be passed to self.cook_training_set
Returns:
self: object
153 def predict(self, X, **kwargs): 154 """Predict test data X. 155 156 Parameters: 157 158 X: {array-like}, shape = [n_samples, n_features] 159 Training vectors, where n_samples is the number 160 of samples and n_features is the number of features 161 162 **kwargs: additional parameters to be passed to self.cook_test_set 163 164 Returns: 165 166 model predictions: {array-like} 167 """ 168 169 if len(X.shape) == 1: 170 n_features = X.shape[0] 171 new_X = mo.rbind( 172 X.reshape(1, n_features), 173 np.ones(n_features).reshape(1, n_features), 174 ) 175 176 return ( 177 self.y_mean_ 178 + mo.safe_sparse_dot( 179 a=self.cook_test_set(new_X, **kwargs), 180 b=self.beta_, 181 backend=self.backend, 182 ) 183 )[0] 184 185 return self.y_mean_ + mo.safe_sparse_dot( 186 a=self.cook_test_set(X, **kwargs), 187 b=self.beta_, 188 backend=self.backend, 189 )
Predict test data X.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
**kwargs: additional parameters to be passed to self.cook_test_set
Returns:
model predictions: {array-like}
191 def score(self, X, y, scoring=None, **kwargs): 192 """Score the model on test set features X and response y. 193 194 Parameters: 195 196 X: {array-like}, shape = [n_samples, n_features] 197 Training vectors, where n_samples is the number 198 of samples and n_features is the number of features 199 200 y: array-like, shape = [n_samples] 201 Target values 202 203 scoring: str 204 must be in ('explained_variance', 'neg_mean_absolute_error', 205 'neg_mean_squared_error', 'neg_mean_squared_log_error', 206 'neg_median_absolute_error', 'r2') 207 208 **kwargs: additional parameters to be passed to scoring functions 209 210 Returns: 211 212 model scores: {array-like} 213 214 """ 215 216 preds = self.predict(X) 217 218 if type(preds) == tuple: # if there are std. devs in the predictions 219 preds = preds[0] 220 221 if scoring is None: 222 scoring = "neg_mean_squared_error" 223 224 # check inputs 225 assert scoring in ( 226 "explained_variance", 227 "neg_mean_absolute_error", 228 "neg_mean_squared_error", 229 "neg_mean_squared_log_error", 230 "neg_median_absolute_error", 231 "r2", 232 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 233 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 234 'neg_median_absolute_error', 'r2')" 235 236 scoring_options = { 237 "explained_variance": skm.explained_variance_score, 238 "neg_mean_absolute_error": skm.median_absolute_error, 239 "neg_mean_squared_error": skm.mean_squared_error, 240 "neg_mean_squared_log_error": skm.mean_squared_log_error, 241 "neg_median_absolute_error": skm.median_absolute_error, 242 "r2": skm.r2_score, 243 } 244 245 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error',
'neg_mean_squared_error', 'neg_mean_squared_log_error',
'neg_median_absolute_error', 'r2')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
15class BayesianRVFLRegressor(Base, RegressorMixin): 16 """Bayesian Random Vector Functional Link Network regression with one prior 17 18 Parameters: 19 20 n_hidden_features: int 21 number of nodes in the hidden layer 22 23 activation_name: str 24 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 25 26 a: float 27 hyperparameter for 'prelu' or 'elu' activation function 28 29 nodes_sim: str 30 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform' 31 32 bias: boolean 33 indicates if the hidden layer contains a bias term (True) or not (False) 34 35 dropout: float 36 regularization parameter; (random) percentage of nodes dropped out 37 of the training 38 39 direct_link: boolean 40 indicates if the original features are included (True) in model''s fitting or not (False) 41 42 n_clusters: int 43 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering) 44 45 cluster_encode: bool 46 defines how the variable containing clusters is treated (default is one-hot) 47 if `False`, then labels are used, without one-hot encoding 48 49 type_clust: str 50 type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm') 51 52 type_scaling: a tuple of 3 strings 53 scaling methods for inputs, hidden layer, and clustering respectively 54 (and when relevant). 55 Currently available: standardization ('std') or MinMax scaling ('minmax') 56 57 seed: int 58 reproducibility seed for nodes_sim=='uniform' 59 60 s: float 61 std. dev. of regression parameters in Bayesian Ridge Regression 62 63 sigma: float 64 std. dev. of residuals in Bayesian Ridge Regression 65 66 return_std: boolean 67 if True, uncertainty around predictions is evaluated 68 69 backend: str 70 "cpu" or "gpu" or "tpu" 71 72 Attributes: 73 74 beta_: array-like 75 regression''s coefficients 76 77 Sigma_: array-like 78 covariance of the distribution of fitted parameters 79 80 GCV_: float 81 Generalized cross-validation error 82 83 y_mean_: float 84 average response 85 86 Examples: 87 88 ```python 89 TBD 90 ``` 91 92 """ 93 94 # construct the object ----- 95 96 def __init__( 97 self, 98 n_hidden_features=5, 99 activation_name="relu", 100 a=0.01, 101 nodes_sim="sobol", 102 bias=True, 103 dropout=0, 104 direct_link=True, 105 n_clusters=2, 106 cluster_encode=True, 107 type_clust="kmeans", 108 type_scaling=("std", "std", "std"), 109 seed=123, 110 s=0.1, 111 sigma=0.05, 112 return_std=True, 113 backend="cpu", 114 ): 115 super().__init__( 116 n_hidden_features=n_hidden_features, 117 activation_name=activation_name, 118 a=a, 119 nodes_sim=nodes_sim, 120 bias=bias, 121 dropout=dropout, 122 direct_link=direct_link, 123 n_clusters=n_clusters, 124 cluster_encode=cluster_encode, 125 type_clust=type_clust, 126 type_scaling=type_scaling, 127 seed=seed, 128 backend=backend, 129 ) 130 self.s = s 131 self.sigma = sigma 132 self.beta_ = None 133 self.Sigma_ = None 134 self.GCV_ = None 135 self.return_std = return_std 136 137 def fit(self, X, y, **kwargs): 138 """Fit BayesianRVFLRegressor to training data (X, y). 139 140 Parameters: 141 142 X: {array-like}, shape = [n_samples, n_features] 143 Training vectors, where n_samples is the number 144 of samples and n_features is the number of features. 145 146 y: array-like, shape = [n_samples] 147 Target values. 148 149 **kwargs: additional parameters to be passed to 150 self.cook_training_set 151 152 Returns: 153 154 self: object 155 156 """ 157 158 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 159 160 fit_obj = lmf.beta_Sigma_hat_rvfl( 161 X=scaled_Z, 162 y=centered_y, 163 s=self.s, 164 sigma=self.sigma, 165 fit_intercept=False, 166 return_cov=self.return_std, 167 backend=self.backend, 168 ) 169 170 self.beta_ = fit_obj["beta_hat"] 171 172 if self.return_std == True: 173 self.Sigma_ = fit_obj["Sigma_hat"] 174 175 self.GCV_ = fit_obj["GCV"] 176 177 return self 178 179 def predict(self, X, return_std=False, **kwargs): 180 """Predict test data X. 181 182 Parameters: 183 184 X: {array-like}, shape = [n_samples, n_features] 185 Training vectors, where n_samples is the number 186 of samples and n_features is the number of features. 187 188 return_std: {boolean}, standard dev. is returned or not 189 190 **kwargs: additional parameters to be passed to 191 self.cook_test_set 192 193 Returns: 194 195 model predictions: {array-like} 196 197 """ 198 199 if len(X.shape) == 1: # one observation in the test set only 200 n_features = X.shape[0] 201 new_X = mo.rbind( 202 x=X.reshape(1, n_features), 203 y=np.ones(n_features).reshape(1, n_features), 204 backend=self.backend, 205 ) 206 207 self.return_std = return_std 208 209 if self.return_std == False: 210 if len(X.shape) == 1: 211 return ( 212 self.y_mean_ 213 + mo.safe_sparse_dot( 214 a=self.cook_test_set(new_X, **kwargs), 215 b=self.beta_, 216 backend=self.backend, 217 ) 218 )[0] 219 220 return self.y_mean_ + mo.safe_sparse_dot( 221 a=self.cook_test_set(X, **kwargs), 222 b=self.beta_, 223 backend=self.backend, 224 ) 225 226 else: # confidence interval required for preds? 227 if len(X.shape) == 1: 228 Z = self.cook_test_set(new_X, **kwargs) 229 230 pred_obj = lmf.beta_Sigma_hat_rvfl( 231 s=self.s, 232 sigma=self.sigma, 233 X_star=Z, 234 return_cov=True, 235 beta_hat_=self.beta_, 236 Sigma_hat_=self.Sigma_, 237 backend=self.backend, 238 ) 239 240 return ( 241 self.y_mean_ + pred_obj["preds"][0], 242 pred_obj["preds_std"][0], 243 ) 244 245 Z = self.cook_test_set(X, **kwargs) 246 247 pred_obj = lmf.beta_Sigma_hat_rvfl( 248 s=self.s, 249 sigma=self.sigma, 250 X_star=Z, 251 return_cov=True, 252 beta_hat_=self.beta_, 253 Sigma_hat_=self.Sigma_, 254 backend=self.backend, 255 ) 256 257 return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"]) 258 259 def score(self, X, y, scoring=None, **kwargs): 260 """ Score the model on test set features X and response y. 261 262 Args: 263 264 X: {array-like}, shape = [n_samples, n_features] 265 Training vectors, where n_samples is the number 266 of samples and n_features is the number of features 267 268 y: array-like, shape = [n_samples] 269 Target values 270 271 scoring: str 272 must be in ('explained_variance', 'neg_mean_absolute_error', \ 273 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 274 'neg_median_absolute_error', 'r2') 275 276 **kwargs: additional parameters to be passed to scoring functions 277 278 Returns: 279 280 model scores: {array-like} 281 282 """ 283 284 preds = self.predict(X) 285 286 if type(preds) == tuple: # if there are std. devs in the predictions 287 preds = preds[0] 288 289 if scoring is None: 290 scoring = "neg_mean_squared_error" 291 292 # check inputs 293 assert scoring in ( 294 "explained_variance", 295 "neg_mean_absolute_error", 296 "neg_mean_squared_error", 297 "neg_mean_squared_log_error", 298 "neg_median_absolute_error", 299 "r2", 300 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 301 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 302 'neg_median_absolute_error', 'r2')" 303 304 scoring_options = { 305 "explained_variance": skm.explained_variance_score, 306 "neg_mean_absolute_error": skm.median_absolute_error, 307 "neg_mean_squared_error": skm.mean_squared_error, 308 "neg_mean_squared_log_error": skm.mean_squared_log_error, 309 "neg_median_absolute_error": skm.median_absolute_error, 310 "r2": skm.r2_score, 311 } 312 313 return scoring_options[scoring](y, preds, **kwargs)
Bayesian Random Vector Functional Link Network regression with one prior
Parameters:
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not (False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original features are included (True) in model''s fitting or not (False)
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
seed: int
reproducibility seed for nodes_sim=='uniform'
s: float
std. dev. of regression parameters in Bayesian Ridge Regression
sigma: float
std. dev. of residuals in Bayesian Ridge Regression
return_std: boolean
if True, uncertainty around predictions is evaluated
backend: str
"cpu" or "gpu" or "tpu"
Attributes:
beta_: array-like
regression''s coefficients
Sigma_: array-like
covariance of the distribution of fitted parameters
GCV_: float
Generalized cross-validation error
y_mean_: float
average response
Examples:
TBD
96 def __init__( 97 self, 98 n_hidden_features=5, 99 activation_name="relu", 100 a=0.01, 101 nodes_sim="sobol", 102 bias=True, 103 dropout=0, 104 direct_link=True, 105 n_clusters=2, 106 cluster_encode=True, 107 type_clust="kmeans", 108 type_scaling=("std", "std", "std"), 109 seed=123, 110 s=0.1, 111 sigma=0.05, 112 return_std=True, 113 backend="cpu", 114 ): 115 super().__init__( 116 n_hidden_features=n_hidden_features, 117 activation_name=activation_name, 118 a=a, 119 nodes_sim=nodes_sim, 120 bias=bias, 121 dropout=dropout, 122 direct_link=direct_link, 123 n_clusters=n_clusters, 124 cluster_encode=cluster_encode, 125 type_clust=type_clust, 126 type_scaling=type_scaling, 127 seed=seed, 128 backend=backend, 129 ) 130 self.s = s 131 self.sigma = sigma 132 self.beta_ = None 133 self.Sigma_ = None 134 self.GCV_ = None 135 self.return_std = return_std
137 def fit(self, X, y, **kwargs): 138 """Fit BayesianRVFLRegressor to training data (X, y). 139 140 Parameters: 141 142 X: {array-like}, shape = [n_samples, n_features] 143 Training vectors, where n_samples is the number 144 of samples and n_features is the number of features. 145 146 y: array-like, shape = [n_samples] 147 Target values. 148 149 **kwargs: additional parameters to be passed to 150 self.cook_training_set 151 152 Returns: 153 154 self: object 155 156 """ 157 158 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 159 160 fit_obj = lmf.beta_Sigma_hat_rvfl( 161 X=scaled_Z, 162 y=centered_y, 163 s=self.s, 164 sigma=self.sigma, 165 fit_intercept=False, 166 return_cov=self.return_std, 167 backend=self.backend, 168 ) 169 170 self.beta_ = fit_obj["beta_hat"] 171 172 if self.return_std == True: 173 self.Sigma_ = fit_obj["Sigma_hat"] 174 175 self.GCV_ = fit_obj["GCV"] 176 177 return self
Fit BayesianRVFLRegressor to training data (X, y).
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set
Returns:
self: object
179 def predict(self, X, return_std=False, **kwargs): 180 """Predict test data X. 181 182 Parameters: 183 184 X: {array-like}, shape = [n_samples, n_features] 185 Training vectors, where n_samples is the number 186 of samples and n_features is the number of features. 187 188 return_std: {boolean}, standard dev. is returned or not 189 190 **kwargs: additional parameters to be passed to 191 self.cook_test_set 192 193 Returns: 194 195 model predictions: {array-like} 196 197 """ 198 199 if len(X.shape) == 1: # one observation in the test set only 200 n_features = X.shape[0] 201 new_X = mo.rbind( 202 x=X.reshape(1, n_features), 203 y=np.ones(n_features).reshape(1, n_features), 204 backend=self.backend, 205 ) 206 207 self.return_std = return_std 208 209 if self.return_std == False: 210 if len(X.shape) == 1: 211 return ( 212 self.y_mean_ 213 + mo.safe_sparse_dot( 214 a=self.cook_test_set(new_X, **kwargs), 215 b=self.beta_, 216 backend=self.backend, 217 ) 218 )[0] 219 220 return self.y_mean_ + mo.safe_sparse_dot( 221 a=self.cook_test_set(X, **kwargs), 222 b=self.beta_, 223 backend=self.backend, 224 ) 225 226 else: # confidence interval required for preds? 227 if len(X.shape) == 1: 228 Z = self.cook_test_set(new_X, **kwargs) 229 230 pred_obj = lmf.beta_Sigma_hat_rvfl( 231 s=self.s, 232 sigma=self.sigma, 233 X_star=Z, 234 return_cov=True, 235 beta_hat_=self.beta_, 236 Sigma_hat_=self.Sigma_, 237 backend=self.backend, 238 ) 239 240 return ( 241 self.y_mean_ + pred_obj["preds"][0], 242 pred_obj["preds_std"][0], 243 ) 244 245 Z = self.cook_test_set(X, **kwargs) 246 247 pred_obj = lmf.beta_Sigma_hat_rvfl( 248 s=self.s, 249 sigma=self.sigma, 250 X_star=Z, 251 return_cov=True, 252 beta_hat_=self.beta_, 253 Sigma_hat_=self.Sigma_, 254 backend=self.backend, 255 ) 256 257 return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"])
Predict test data X.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
return_std: {boolean}, standard dev. is returned or not
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
259 def score(self, X, y, scoring=None, **kwargs): 260 """ Score the model on test set features X and response y. 261 262 Args: 263 264 X: {array-like}, shape = [n_samples, n_features] 265 Training vectors, where n_samples is the number 266 of samples and n_features is the number of features 267 268 y: array-like, shape = [n_samples] 269 Target values 270 271 scoring: str 272 must be in ('explained_variance', 'neg_mean_absolute_error', \ 273 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 274 'neg_median_absolute_error', 'r2') 275 276 **kwargs: additional parameters to be passed to scoring functions 277 278 Returns: 279 280 model scores: {array-like} 281 282 """ 283 284 preds = self.predict(X) 285 286 if type(preds) == tuple: # if there are std. devs in the predictions 287 preds = preds[0] 288 289 if scoring is None: 290 scoring = "neg_mean_squared_error" 291 292 # check inputs 293 assert scoring in ( 294 "explained_variance", 295 "neg_mean_absolute_error", 296 "neg_mean_squared_error", 297 "neg_mean_squared_log_error", 298 "neg_median_absolute_error", 299 "r2", 300 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 301 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 302 'neg_median_absolute_error', 'r2')" 303 304 scoring_options = { 305 "explained_variance": skm.explained_variance_score, 306 "neg_mean_absolute_error": skm.median_absolute_error, 307 "neg_mean_squared_error": skm.mean_squared_error, 308 "neg_mean_squared_log_error": skm.mean_squared_log_error, 309 "neg_median_absolute_error": skm.median_absolute_error, 310 "r2": skm.r2_score, 311 } 312 313 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
15class BayesianRVFL2Regressor(Base, RegressorMixin): 16 """Bayesian Random Vector Functional Link Network regression with two priors 17 18 Parameters: 19 20 n_hidden_features: int 21 number of nodes in the hidden layer 22 23 activation_name: str 24 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 25 26 a: float 27 hyperparameter for 'prelu' or 'elu' activation function 28 29 nodes_sim: str 30 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform' 31 32 bias: boolean 33 indicates if the hidden layer contains a bias term (True) or not (False) 34 35 dropout: float 36 regularization parameter; (random) percentage of nodes dropped out 37 of the training 38 39 direct_link: boolean 40 indicates if the original features are included (True) in model''s fitting or not (False) 41 42 n_clusters: int 43 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering) 44 45 cluster_encode: bool 46 defines how the variable containing clusters is treated (default is one-hot) 47 if `False`, then labels are used, without one-hot encoding 48 49 type_clust: str 50 type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm') 51 52 type_scaling: a tuple of 3 strings 53 scaling methods for inputs, hidden layer, and clustering respectively 54 (and when relevant). 55 Currently available: standardization ('std') or MinMax scaling ('minmax') 56 57 seed: int 58 reproducibility seed for nodes_sim=='uniform' 59 60 s1: float 61 std. dev. of init. regression parameters in Bayesian Ridge Regression 62 63 s2: float 64 std. dev. of augmented regression parameters in Bayesian Ridge Regression 65 66 sigma: float 67 std. dev. of residuals in Bayesian Ridge Regression 68 69 return_std: boolean 70 if True, uncertainty around predictions is evaluated 71 72 backend: str 73 "cpu" or "gpu" or "tpu" 74 75 Attributes: 76 77 beta_: array-like 78 regression''s coefficients 79 80 Sigma_: array-like 81 covariance of the distribution of fitted parameters 82 83 GCV_: float 84 Generalized cross-validation error 85 86 y_mean_: float 87 average response 88 89 Examples: 90 91 ```python 92 TBD 93 ``` 94 95 """ 96 97 # construct the object ----- 98 99 def __init__( 100 self, 101 n_hidden_features=5, 102 activation_name="relu", 103 a=0.01, 104 nodes_sim="sobol", 105 bias=True, 106 dropout=0, 107 direct_link=True, 108 n_clusters=0, 109 cluster_encode=True, 110 type_clust="kmeans", 111 type_scaling=("std", "std", "std"), 112 seed=123, 113 s1=0.1, 114 s2=0.1, 115 sigma=0.05, 116 return_std=True, 117 backend="cpu", 118 ): 119 super().__init__( 120 n_hidden_features=n_hidden_features, 121 activation_name=activation_name, 122 a=a, 123 nodes_sim=nodes_sim, 124 bias=bias, 125 dropout=dropout, 126 direct_link=direct_link, 127 n_clusters=n_clusters, 128 cluster_encode=cluster_encode, 129 type_clust=type_clust, 130 type_scaling=type_scaling, 131 seed=seed, 132 backend=backend, 133 ) 134 135 self.s1 = s1 136 self.s2 = s2 137 self.sigma = sigma 138 self.beta_ = None 139 self.Sigma_ = None 140 self.GCV_ = None 141 self.return_std = return_std 142 143 def fit(self, X, y, **kwargs): 144 """Fit BayesianRVFL2Regressor to training data (X, y) 145 146 Parameters: 147 148 X: {array-like}, shape = [n_samples, n_features] 149 Training vectors, where n_samples is the number 150 of samples and n_features is the number of features 151 152 y: array-like, shape = [n_samples] 153 Target values 154 155 **kwargs: additional parameters to be passed to 156 self.cook_training_set 157 158 Returns: 159 160 self: object 161 162 """ 163 164 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 165 166 n, p = X.shape 167 q = self.n_hidden_features 168 169 if self.direct_link == True: 170 r = p + self.n_clusters 171 172 block11 = (self.s1**2) * np.eye(r) 173 block12 = np.zeros((r, q)) 174 block21 = np.zeros((q, r)) 175 block22 = (self.s2**2) * np.eye(q) 176 177 Sigma_prior = mo.rbind( 178 x=mo.cbind(x=block11, y=block12, backend=self.backend), 179 y=mo.cbind(x=block21, y=block22, backend=self.backend), 180 backend=self.backend, 181 ) 182 183 else: 184 Sigma_prior = (self.s2**2) * np.eye(q) 185 186 fit_obj = lmf.beta_Sigma_hat_rvfl2( 187 X=scaled_Z, 188 y=centered_y, 189 Sigma=Sigma_prior, 190 sigma=self.sigma, 191 fit_intercept=False, 192 return_cov=self.return_std, 193 backend=self.backend, 194 ) 195 196 self.beta_ = fit_obj["beta_hat"] 197 198 if self.return_std == True: 199 self.Sigma_ = fit_obj["Sigma_hat"] 200 201 self.GCV_ = fit_obj["GCV"] 202 203 return self 204 205 def predict(self, X, return_std=False, **kwargs): 206 """Predict test data X. 207 208 Parameters: 209 210 X: {array-like}, shape = [n_samples, n_features] 211 Training vectors, where n_samples is the number 212 of samples and n_features is the number of features. 213 214 return_std: {boolean}, standard dev. is returned or not 215 216 **kwargs: additional parameters to be passed to 217 self.cook_test_set 218 219 Returns: 220 221 model predictions: {array-like} 222 223 """ 224 225 if len(X.shape) == 1: # one observation in the test set only 226 n_features = X.shape[0] 227 new_X = mo.rbind( 228 x=X.reshape(1, n_features), 229 y=np.ones(n_features).reshape(1, n_features), 230 backend=self.backend, 231 ) 232 233 self.return_std = return_std 234 235 if self.return_std == False: 236 if len(X.shape) == 1: 237 return ( 238 self.y_mean_ 239 + mo.safe_sparse_dot( 240 self.cook_test_set(new_X, **kwargs), 241 self.beta_, 242 backend=self.backend, 243 ) 244 )[0] 245 246 return self.y_mean_ + mo.safe_sparse_dot( 247 self.cook_test_set(X, **kwargs), 248 self.beta_, 249 backend=self.backend, 250 ) 251 252 else: # confidence interval required for preds? 253 if len(X.shape) == 1: 254 Z = self.cook_test_set(new_X, **kwargs) 255 256 pred_obj = lmf.beta_Sigma_hat_rvfl2( 257 X_star=Z, 258 return_cov=self.return_std, 259 beta_hat_=self.beta_, 260 Sigma_hat_=self.Sigma_, 261 backend=self.backend, 262 ) 263 264 return ( 265 self.y_mean_ + pred_obj["preds"][0], 266 pred_obj["preds_std"][0], 267 ) 268 269 Z = self.cook_test_set(X, **kwargs) 270 271 pred_obj = lmf.beta_Sigma_hat_rvfl2( 272 X_star=Z, 273 return_cov=self.return_std, 274 beta_hat_=self.beta_, 275 Sigma_hat_=self.Sigma_, 276 backend=self.backend, 277 ) 278 279 return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"]) 280 281 def score(self, X, y, scoring=None, **kwargs): 282 """ Score the model on test set features X and response y. 283 284 Args: 285 286 X: {array-like}, shape = [n_samples, n_features] 287 Training vectors, where n_samples is the number 288 of samples and n_features is the number of features 289 290 y: array-like, shape = [n_samples] 291 Target values 292 293 scoring: str 294 must be in ('explained_variance', 'neg_mean_absolute_error', \ 295 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 296 'neg_median_absolute_error', 'r2') 297 298 **kwargs: additional parameters to be passed to scoring functions 299 300 Returns: 301 302 model scores: {array-like} 303 304 """ 305 306 preds = self.predict(X) 307 308 if type(preds) == tuple: # if there are std. devs in the predictions 309 preds = preds[0] 310 311 if scoring is None: 312 scoring = "neg_mean_squared_error" 313 314 # check inputs 315 assert scoring in ( 316 "explained_variance", 317 "neg_mean_absolute_error", 318 "neg_mean_squared_error", 319 "neg_mean_squared_log_error", 320 "neg_median_absolute_error", 321 "r2", 322 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 323 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 324 'neg_median_absolute_error', 'r2')" 325 326 scoring_options = { 327 "explained_variance": skm.explained_variance_score, 328 "neg_mean_absolute_error": skm.median_absolute_error, 329 "neg_mean_squared_error": skm.mean_squared_error, 330 "neg_mean_squared_log_error": skm.mean_squared_log_error, 331 "neg_median_absolute_error": skm.median_absolute_error, 332 "r2": skm.r2_score, 333 } 334 335 return scoring_options[scoring](y, preds, **kwargs)
Bayesian Random Vector Functional Link Network regression with two priors
Parameters:
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not (False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original features are included (True) in model''s fitting or not (False)
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
seed: int
reproducibility seed for nodes_sim=='uniform'
s1: float
std. dev. of init. regression parameters in Bayesian Ridge Regression
s2: float
std. dev. of augmented regression parameters in Bayesian Ridge Regression
sigma: float
std. dev. of residuals in Bayesian Ridge Regression
return_std: boolean
if True, uncertainty around predictions is evaluated
backend: str
"cpu" or "gpu" or "tpu"
Attributes:
beta_: array-like
regression''s coefficients
Sigma_: array-like
covariance of the distribution of fitted parameters
GCV_: float
Generalized cross-validation error
y_mean_: float
average response
Examples:
TBD
99 def __init__( 100 self, 101 n_hidden_features=5, 102 activation_name="relu", 103 a=0.01, 104 nodes_sim="sobol", 105 bias=True, 106 dropout=0, 107 direct_link=True, 108 n_clusters=0, 109 cluster_encode=True, 110 type_clust="kmeans", 111 type_scaling=("std", "std", "std"), 112 seed=123, 113 s1=0.1, 114 s2=0.1, 115 sigma=0.05, 116 return_std=True, 117 backend="cpu", 118 ): 119 super().__init__( 120 n_hidden_features=n_hidden_features, 121 activation_name=activation_name, 122 a=a, 123 nodes_sim=nodes_sim, 124 bias=bias, 125 dropout=dropout, 126 direct_link=direct_link, 127 n_clusters=n_clusters, 128 cluster_encode=cluster_encode, 129 type_clust=type_clust, 130 type_scaling=type_scaling, 131 seed=seed, 132 backend=backend, 133 ) 134 135 self.s1 = s1 136 self.s2 = s2 137 self.sigma = sigma 138 self.beta_ = None 139 self.Sigma_ = None 140 self.GCV_ = None 141 self.return_std = return_std
143 def fit(self, X, y, **kwargs): 144 """Fit BayesianRVFL2Regressor to training data (X, y) 145 146 Parameters: 147 148 X: {array-like}, shape = [n_samples, n_features] 149 Training vectors, where n_samples is the number 150 of samples and n_features is the number of features 151 152 y: array-like, shape = [n_samples] 153 Target values 154 155 **kwargs: additional parameters to be passed to 156 self.cook_training_set 157 158 Returns: 159 160 self: object 161 162 """ 163 164 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 165 166 n, p = X.shape 167 q = self.n_hidden_features 168 169 if self.direct_link == True: 170 r = p + self.n_clusters 171 172 block11 = (self.s1**2) * np.eye(r) 173 block12 = np.zeros((r, q)) 174 block21 = np.zeros((q, r)) 175 block22 = (self.s2**2) * np.eye(q) 176 177 Sigma_prior = mo.rbind( 178 x=mo.cbind(x=block11, y=block12, backend=self.backend), 179 y=mo.cbind(x=block21, y=block22, backend=self.backend), 180 backend=self.backend, 181 ) 182 183 else: 184 Sigma_prior = (self.s2**2) * np.eye(q) 185 186 fit_obj = lmf.beta_Sigma_hat_rvfl2( 187 X=scaled_Z, 188 y=centered_y, 189 Sigma=Sigma_prior, 190 sigma=self.sigma, 191 fit_intercept=False, 192 return_cov=self.return_std, 193 backend=self.backend, 194 ) 195 196 self.beta_ = fit_obj["beta_hat"] 197 198 if self.return_std == True: 199 self.Sigma_ = fit_obj["Sigma_hat"] 200 201 self.GCV_ = fit_obj["GCV"] 202 203 return self
Fit BayesianRVFL2Regressor to training data (X, y)
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
**kwargs: additional parameters to be passed to
self.cook_training_set
Returns:
self: object
205 def predict(self, X, return_std=False, **kwargs): 206 """Predict test data X. 207 208 Parameters: 209 210 X: {array-like}, shape = [n_samples, n_features] 211 Training vectors, where n_samples is the number 212 of samples and n_features is the number of features. 213 214 return_std: {boolean}, standard dev. is returned or not 215 216 **kwargs: additional parameters to be passed to 217 self.cook_test_set 218 219 Returns: 220 221 model predictions: {array-like} 222 223 """ 224 225 if len(X.shape) == 1: # one observation in the test set only 226 n_features = X.shape[0] 227 new_X = mo.rbind( 228 x=X.reshape(1, n_features), 229 y=np.ones(n_features).reshape(1, n_features), 230 backend=self.backend, 231 ) 232 233 self.return_std = return_std 234 235 if self.return_std == False: 236 if len(X.shape) == 1: 237 return ( 238 self.y_mean_ 239 + mo.safe_sparse_dot( 240 self.cook_test_set(new_X, **kwargs), 241 self.beta_, 242 backend=self.backend, 243 ) 244 )[0] 245 246 return self.y_mean_ + mo.safe_sparse_dot( 247 self.cook_test_set(X, **kwargs), 248 self.beta_, 249 backend=self.backend, 250 ) 251 252 else: # confidence interval required for preds? 253 if len(X.shape) == 1: 254 Z = self.cook_test_set(new_X, **kwargs) 255 256 pred_obj = lmf.beta_Sigma_hat_rvfl2( 257 X_star=Z, 258 return_cov=self.return_std, 259 beta_hat_=self.beta_, 260 Sigma_hat_=self.Sigma_, 261 backend=self.backend, 262 ) 263 264 return ( 265 self.y_mean_ + pred_obj["preds"][0], 266 pred_obj["preds_std"][0], 267 ) 268 269 Z = self.cook_test_set(X, **kwargs) 270 271 pred_obj = lmf.beta_Sigma_hat_rvfl2( 272 X_star=Z, 273 return_cov=self.return_std, 274 beta_hat_=self.beta_, 275 Sigma_hat_=self.Sigma_, 276 backend=self.backend, 277 ) 278 279 return (self.y_mean_ + pred_obj["preds"], pred_obj["preds_std"])
Predict test data X.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
return_std: {boolean}, standard dev. is returned or not
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
281 def score(self, X, y, scoring=None, **kwargs): 282 """ Score the model on test set features X and response y. 283 284 Args: 285 286 X: {array-like}, shape = [n_samples, n_features] 287 Training vectors, where n_samples is the number 288 of samples and n_features is the number of features 289 290 y: array-like, shape = [n_samples] 291 Target values 292 293 scoring: str 294 must be in ('explained_variance', 'neg_mean_absolute_error', \ 295 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 296 'neg_median_absolute_error', 'r2') 297 298 **kwargs: additional parameters to be passed to scoring functions 299 300 Returns: 301 302 model scores: {array-like} 303 304 """ 305 306 preds = self.predict(X) 307 308 if type(preds) == tuple: # if there are std. devs in the predictions 309 preds = preds[0] 310 311 if scoring is None: 312 scoring = "neg_mean_squared_error" 313 314 # check inputs 315 assert scoring in ( 316 "explained_variance", 317 "neg_mean_absolute_error", 318 "neg_mean_squared_error", 319 "neg_mean_squared_log_error", 320 "neg_median_absolute_error", 321 "r2", 322 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 323 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 324 'neg_median_absolute_error', 'r2')" 325 326 scoring_options = { 327 "explained_variance": skm.explained_variance_score, 328 "neg_mean_absolute_error": skm.median_absolute_error, 329 "neg_mean_squared_error": skm.mean_squared_error, 330 "neg_mean_squared_log_error": skm.mean_squared_log_error, 331 "neg_median_absolute_error": skm.median_absolute_error, 332 "r2": skm.r2_score, 333 } 334 335 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
13class CustomClassifier(Custom, ClassifierMixin): 14 """Custom Classification model 15 16 Attributes: 17 18 obj: object 19 any object containing a method fit (obj.fit()) and a method predict 20 (obj.predict()) 21 22 n_hidden_features: int 23 number of nodes in the hidden layer 24 25 activation_name: str 26 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 27 28 a: float 29 hyperparameter for 'prelu' or 'elu' activation function 30 31 nodes_sim: str 32 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 33 'uniform' 34 35 bias: boolean 36 indicates if the hidden layer contains a bias term (True) or not 37 (False) 38 39 dropout: float 40 regularization parameter; (random) percentage of nodes dropped out 41 of the training 42 43 direct_link: boolean 44 indicates if the original predictors are included (True) in model''s 45 fitting or not (False) 46 47 n_clusters: int 48 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: 49 no clustering) 50 51 cluster_encode: bool 52 defines how the variable containing clusters is treated (default is one-hot) 53 if `False`, then labels are used, without one-hot encoding 54 55 type_clust: str 56 type of clustering method: currently k-means ('kmeans') or Gaussian 57 Mixture Model ('gmm') 58 59 type_scaling: a tuple of 3 strings 60 scaling methods for inputs, hidden layer, and clustering respectively 61 (and when relevant). 62 Currently available: standardization ('std') or MinMax scaling ('minmax') 63 64 col_sample: float 65 percentage of covariates randomly chosen for training 66 67 row_sample: float 68 percentage of rows chosen for training, by stratified bootstrapping 69 70 seed: int 71 reproducibility seed for nodes_sim=='uniform' 72 73 backend: str 74 "cpu" or "gpu" or "tpu" 75 76 Examples: 77 78 ```python 79 import nnetsauce as ns 80 from sklearn.ensemble import RandomForestClassifier 81 from sklearn.model_selection import train_test_split 82 from sklearn.datasets import load_digits 83 from time import time 84 85 digits = load_digits() 86 X = digits.data 87 y = digits.target 88 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, 89 random_state=123) 90 91 # layer 1 (base layer) ---- 92 layer1_regr = RandomForestClassifier(n_estimators=10, random_state=123) 93 94 start = time() 95 96 layer1_regr.fit(X_train, y_train) 97 98 # Accuracy in layer 1 99 print(layer1_regr.score(X_test, y_test)) 100 101 # layer 2 using layer 1 ---- 102 layer2_regr = ns.CustomClassifier(obj = layer1_regr, n_hidden_features=5, 103 direct_link=True, bias=True, 104 nodes_sim='uniform', activation_name='relu', 105 n_clusters=2, seed=123) 106 layer2_regr.fit(X_train, y_train) 107 108 # Accuracy in layer 2 109 print(layer2_regr.score(X_test, y_test)) 110 111 # layer 3 using layer 2 ---- 112 layer3_regr = ns.CustomClassifier(obj = layer2_regr, n_hidden_features=10, 113 direct_link=True, bias=True, dropout=0.7, 114 nodes_sim='uniform', activation_name='relu', 115 n_clusters=2, seed=123) 116 layer3_regr.fit(X_train, y_train) 117 118 # Accuracy in layer 3 119 print(layer3_regr.score(X_test, y_test)) 120 121 print(f"Elapsed {time() - start}") 122 ``` 123 124 """ 125 126 # construct the object ----- 127 128 def __init__( 129 self, 130 obj, 131 n_hidden_features=5, 132 activation_name="relu", 133 a=0.01, 134 nodes_sim="sobol", 135 bias=True, 136 dropout=0, 137 direct_link=True, 138 n_clusters=2, 139 cluster_encode=True, 140 type_clust="kmeans", 141 type_scaling=("std", "std", "std"), 142 col_sample=1, 143 row_sample=1, 144 seed=123, 145 backend="cpu", 146 ): 147 super().__init__( 148 obj=obj, 149 n_hidden_features=n_hidden_features, 150 activation_name=activation_name, 151 a=a, 152 nodes_sim=nodes_sim, 153 bias=bias, 154 dropout=dropout, 155 direct_link=direct_link, 156 n_clusters=n_clusters, 157 cluster_encode=cluster_encode, 158 type_clust=type_clust, 159 type_scaling=type_scaling, 160 col_sample=col_sample, 161 row_sample=row_sample, 162 seed=seed, 163 backend=backend, 164 ) 165 166 self.type_fit = "classification" 167 168 def fit(self, X, y, sample_weight=None, **kwargs): 169 """Fit custom model to training data (X, y). 170 171 Parameters: 172 173 X: {array-like}, shape = [n_samples, n_features] 174 Training vectors, where n_samples is the number 175 of samples and n_features is the number of features. 176 177 y: array-like, shape = [n_samples] 178 Target values. 179 180 **kwargs: additional parameters to be passed to 181 self.cook_training_set or self.obj.fit 182 183 Returns: 184 185 self: object 186 """ 187 188 output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 189 190 # if sample_weights, else: (must use self.row_index) 191 if sample_weight is not None: 192 self.obj.fit( 193 scaled_Z, 194 output_y, 195 sample_weight=np.ravel(sample_weight, order="C")[ 196 self.index_row_ 197 ], 198 # **kwargs 199 ) 200 201 return self 202 203 # if sample_weight is None: 204 self.obj.fit(scaled_Z, output_y) 205 206 return self 207 208 def predict(self, X, **kwargs): 209 """Predict test data X. 210 211 Parameters: 212 213 X: {array-like}, shape = [n_samples, n_features] 214 Training vectors, where n_samples is the number 215 of samples and n_features is the number of features. 216 217 **kwargs: additional parameters to be passed to 218 self.cook_test_set 219 220 Returns: 221 222 model predictions: {array-like} 223 """ 224 225 if len(X.shape) == 1: 226 n_features = X.shape[0] 227 new_X = mo.rbind( 228 X.reshape(1, n_features), 229 np.ones(n_features).reshape(1, n_features), 230 ) 231 232 return ( 233 self.obj.predict(self.cook_test_set(new_X, **kwargs), **kwargs) 234 )[0] 235 236 return self.obj.predict(self.cook_test_set(X, **kwargs), **kwargs) 237 238 def predict_proba(self, X, **kwargs): 239 """Predict probabilities for test data X. 240 241 Args: 242 243 X: {array-like}, shape = [n_samples, n_features] 244 Training vectors, where n_samples is the number 245 of samples and n_features is the number of features. 246 247 **kwargs: additional parameters to be passed to 248 self.cook_test_set 249 250 Returns: 251 252 probability estimates for test data: {array-like} 253 """ 254 255 if len(X.shape) == 1: 256 n_features = X.shape[0] 257 new_X = mo.rbind( 258 X.reshape(1, n_features), 259 np.ones(n_features).reshape(1, n_features), 260 ) 261 262 return ( 263 self.obj.predict_proba( 264 self.cook_test_set(new_X, **kwargs), **kwargs 265 ) 266 )[0] 267 268 return self.obj.predict_proba(self.cook_test_set(X, **kwargs), **kwargs) 269 270 def score(self, X, y, scoring=None, **kwargs): 271 """ Score the model on test set features X and response y. 272 273 Args: 274 275 X: {array-like}, shape = [n_samples, n_features] 276 Training vectors, where n_samples is the number 277 of samples and n_features is the number of features 278 279 y: array-like, shape = [n_samples] 280 Target values 281 282 scoring: str 283 must be in ('explained_variance', 'neg_mean_absolute_error', \ 284 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 285 'neg_median_absolute_error', 'r2') 286 287 **kwargs: 288 additional parameters to be passed to scoring functions 289 290 Returns: 291 292 model scores: {array-like} 293 294 """ 295 296 preds = self.predict(X) 297 298 if scoring is None: 299 scoring = "accuracy" 300 301 # check inputs 302 assert scoring in ( 303 "accuracy", 304 "average_precision", 305 "brier_score_loss", 306 "f1", 307 "f1_micro", 308 "f1_macro", 309 "f1_weighted", 310 "f1_samples", 311 "neg_log_loss", 312 "precision", 313 "recall", 314 "roc_auc", 315 ), "'scoring' should be in ('accuracy', 'average_precision', \ 316 'brier_score_loss', 'f1', 'f1_micro', \ 317 'f1_macro', 'f1_weighted', 'f1_samples', \ 318 'neg_log_loss', 'precision', 'recall', \ 319 'roc_auc')" 320 321 scoring_options = { 322 "accuracy": skm2.accuracy_score, 323 "average_precision": skm2.average_precision_score, 324 "brier_score_loss": skm2.brier_score_loss, 325 "f1": skm2.f1_score, 326 "f1_micro": skm2.f1_score, 327 "f1_macro": skm2.f1_score, 328 "f1_weighted": skm2.f1_score, 329 "f1_samples": skm2.f1_score, 330 "neg_log_loss": skm2.log_loss, 331 "precision": skm2.precision_score, 332 "recall": skm2.recall_score, 333 "roc_auc": skm2.roc_auc_score, 334 } 335 336 return scoring_options[scoring](y, preds, **kwargs)
Custom Classification model
Attributes:
obj: object
any object containing a method fit (obj.fit()) and a method predict
(obj.predict())
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not
(False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original predictors are included (True) in model''s
fitting or not (False)
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
col_sample: float
percentage of covariates randomly chosen for training
row_sample: float
percentage of rows chosen for training, by stratified bootstrapping
seed: int
reproducibility seed for nodes_sim=='uniform'
backend: str
"cpu" or "gpu" or "tpu"
Examples:
import nnetsauce as ns
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_digits
from time import time
digits = load_digits()
X = digits.data
y = digits.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
random_state=123)
# layer 1 (base layer) ----
layer1_regr = RandomForestClassifier(n_estimators=10, random_state=123)
start = time()
layer1_regr.fit(X_train, y_train)
# Accuracy in layer 1
print(layer1_regr.score(X_test, y_test))
# layer 2 using layer 1 ----
layer2_regr = ns.CustomClassifier(obj = layer1_regr, n_hidden_features=5,
direct_link=True, bias=True,
nodes_sim='uniform', activation_name='relu',
n_clusters=2, seed=123)
layer2_regr.fit(X_train, y_train)
# Accuracy in layer 2
print(layer2_regr.score(X_test, y_test))
# layer 3 using layer 2 ----
layer3_regr = ns.CustomClassifier(obj = layer2_regr, n_hidden_features=10,
direct_link=True, bias=True, dropout=0.7,
nodes_sim='uniform', activation_name='relu',
n_clusters=2, seed=123)
layer3_regr.fit(X_train, y_train)
# Accuracy in layer 3
print(layer3_regr.score(X_test, y_test))
print(f"Elapsed {time() - start}")
128 def __init__( 129 self, 130 obj, 131 n_hidden_features=5, 132 activation_name="relu", 133 a=0.01, 134 nodes_sim="sobol", 135 bias=True, 136 dropout=0, 137 direct_link=True, 138 n_clusters=2, 139 cluster_encode=True, 140 type_clust="kmeans", 141 type_scaling=("std", "std", "std"), 142 col_sample=1, 143 row_sample=1, 144 seed=123, 145 backend="cpu", 146 ): 147 super().__init__( 148 obj=obj, 149 n_hidden_features=n_hidden_features, 150 activation_name=activation_name, 151 a=a, 152 nodes_sim=nodes_sim, 153 bias=bias, 154 dropout=dropout, 155 direct_link=direct_link, 156 n_clusters=n_clusters, 157 cluster_encode=cluster_encode, 158 type_clust=type_clust, 159 type_scaling=type_scaling, 160 col_sample=col_sample, 161 row_sample=row_sample, 162 seed=seed, 163 backend=backend, 164 ) 165 166 self.type_fit = "classification"
168 def fit(self, X, y, sample_weight=None, **kwargs): 169 """Fit custom model to training data (X, y). 170 171 Parameters: 172 173 X: {array-like}, shape = [n_samples, n_features] 174 Training vectors, where n_samples is the number 175 of samples and n_features is the number of features. 176 177 y: array-like, shape = [n_samples] 178 Target values. 179 180 **kwargs: additional parameters to be passed to 181 self.cook_training_set or self.obj.fit 182 183 Returns: 184 185 self: object 186 """ 187 188 output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 189 190 # if sample_weights, else: (must use self.row_index) 191 if sample_weight is not None: 192 self.obj.fit( 193 scaled_Z, 194 output_y, 195 sample_weight=np.ravel(sample_weight, order="C")[ 196 self.index_row_ 197 ], 198 # **kwargs 199 ) 200 201 return self 202 203 # if sample_weight is None: 204 self.obj.fit(scaled_Z, output_y) 205 206 return self
Fit custom model to training data (X, y).
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
208 def predict(self, X, **kwargs): 209 """Predict test data X. 210 211 Parameters: 212 213 X: {array-like}, shape = [n_samples, n_features] 214 Training vectors, where n_samples is the number 215 of samples and n_features is the number of features. 216 217 **kwargs: additional parameters to be passed to 218 self.cook_test_set 219 220 Returns: 221 222 model predictions: {array-like} 223 """ 224 225 if len(X.shape) == 1: 226 n_features = X.shape[0] 227 new_X = mo.rbind( 228 X.reshape(1, n_features), 229 np.ones(n_features).reshape(1, n_features), 230 ) 231 232 return ( 233 self.obj.predict(self.cook_test_set(new_X, **kwargs), **kwargs) 234 )[0] 235 236 return self.obj.predict(self.cook_test_set(X, **kwargs), **kwargs)
Predict test data X.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
238 def predict_proba(self, X, **kwargs): 239 """Predict probabilities for test data X. 240 241 Args: 242 243 X: {array-like}, shape = [n_samples, n_features] 244 Training vectors, where n_samples is the number 245 of samples and n_features is the number of features. 246 247 **kwargs: additional parameters to be passed to 248 self.cook_test_set 249 250 Returns: 251 252 probability estimates for test data: {array-like} 253 """ 254 255 if len(X.shape) == 1: 256 n_features = X.shape[0] 257 new_X = mo.rbind( 258 X.reshape(1, n_features), 259 np.ones(n_features).reshape(1, n_features), 260 ) 261 262 return ( 263 self.obj.predict_proba( 264 self.cook_test_set(new_X, **kwargs), **kwargs 265 ) 266 )[0] 267 268 return self.obj.predict_proba(self.cook_test_set(X, **kwargs), **kwargs)
Predict probabilities for test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
probability estimates for test data: {array-like}
270 def score(self, X, y, scoring=None, **kwargs): 271 """ Score the model on test set features X and response y. 272 273 Args: 274 275 X: {array-like}, shape = [n_samples, n_features] 276 Training vectors, where n_samples is the number 277 of samples and n_features is the number of features 278 279 y: array-like, shape = [n_samples] 280 Target values 281 282 scoring: str 283 must be in ('explained_variance', 'neg_mean_absolute_error', \ 284 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 285 'neg_median_absolute_error', 'r2') 286 287 **kwargs: 288 additional parameters to be passed to scoring functions 289 290 Returns: 291 292 model scores: {array-like} 293 294 """ 295 296 preds = self.predict(X) 297 298 if scoring is None: 299 scoring = "accuracy" 300 301 # check inputs 302 assert scoring in ( 303 "accuracy", 304 "average_precision", 305 "brier_score_loss", 306 "f1", 307 "f1_micro", 308 "f1_macro", 309 "f1_weighted", 310 "f1_samples", 311 "neg_log_loss", 312 "precision", 313 "recall", 314 "roc_auc", 315 ), "'scoring' should be in ('accuracy', 'average_precision', \ 316 'brier_score_loss', 'f1', 'f1_micro', \ 317 'f1_macro', 'f1_weighted', 'f1_samples', \ 318 'neg_log_loss', 'precision', 'recall', \ 319 'roc_auc')" 320 321 scoring_options = { 322 "accuracy": skm2.accuracy_score, 323 "average_precision": skm2.average_precision_score, 324 "brier_score_loss": skm2.brier_score_loss, 325 "f1": skm2.f1_score, 326 "f1_micro": skm2.f1_score, 327 "f1_macro": skm2.f1_score, 328 "f1_weighted": skm2.f1_score, 329 "f1_samples": skm2.f1_score, 330 "neg_log_loss": skm2.log_loss, 331 "precision": skm2.precision_score, 332 "recall": skm2.recall_score, 333 "roc_auc": skm2.roc_auc_score, 334 } 335 336 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs:
additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.custom.custom.Custom
- obj
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
14class CustomRegressor(Custom, RegressorMixin): 15 """Custom Regression model 16 17 This class is used to 'augment' any regression model with transformed features. 18 19 Parameters: 20 21 obj: object 22 any object containing a method fit (obj.fit()) and a method predict 23 (obj.predict()) 24 25 n_hidden_features: int 26 number of nodes in the hidden layer 27 28 activation_name: str 29 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 30 31 a: float 32 hyperparameter for 'prelu' or 'elu' activation function 33 34 nodes_sim: str 35 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 36 'uniform' 37 38 bias: boolean 39 indicates if the hidden layer contains a bias term (True) or not 40 (False) 41 42 dropout: float 43 regularization parameter; (random) percentage of nodes dropped out 44 of the training 45 46 direct_link: boolean 47 indicates if the original predictors are included (True) in model's 48 fitting or not (False) 49 50 n_clusters: int 51 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: 52 no clustering) 53 54 cluster_encode: bool 55 defines how the variable containing clusters is treated (default is one-hot) 56 if `False`, then labels are used, without one-hot encoding 57 58 type_clust: str 59 type of clustering method: currently k-means ('kmeans') or Gaussian 60 Mixture Model ('gmm') 61 62 type_scaling: a tuple of 3 strings 63 scaling methods for inputs, hidden layer, and clustering respectively 64 (and when relevant). 65 Currently available: standardization ('std') or MinMax scaling ('minmax') 66 67 col_sample: float 68 percentage of covariates randomly chosen for training 69 70 row_sample: float 71 percentage of rows chosen for training, by stratified bootstrapping 72 73 seed: int 74 reproducibility seed for nodes_sim=='uniform' 75 76 type_fit: str 77 'regression' 78 79 backend: str 80 "cpu" or "gpu" or "tpu" 81 82 Examples: 83 84 ```python 85 TBD 86 ``` 87 88 """ 89 90 # construct the object ----- 91 92 def __init__( 93 self, 94 obj, 95 n_hidden_features=5, 96 activation_name="relu", 97 a=0.01, 98 nodes_sim="sobol", 99 bias=True, 100 dropout=0, 101 direct_link=True, 102 n_clusters=2, 103 cluster_encode=True, 104 type_clust="kmeans", 105 type_scaling=("std", "std", "std"), 106 col_sample=1, 107 row_sample=1, 108 seed=123, 109 backend="cpu", 110 ): 111 super().__init__( 112 obj=obj, 113 n_hidden_features=n_hidden_features, 114 activation_name=activation_name, 115 a=a, 116 nodes_sim=nodes_sim, 117 bias=bias, 118 dropout=dropout, 119 direct_link=direct_link, 120 n_clusters=n_clusters, 121 cluster_encode=cluster_encode, 122 type_clust=type_clust, 123 type_scaling=type_scaling, 124 col_sample=col_sample, 125 row_sample=row_sample, 126 seed=seed, 127 backend=backend, 128 ) 129 130 self.type_fit = "regression" 131 132 def fit(self, X, y, sample_weight=None, **kwargs): 133 """Fit custom model to training data (X, y). 134 135 Parameters: 136 137 X: {array-like}, shape = [n_samples, n_features] 138 Training vectors, where n_samples is the number 139 of samples and n_features is the number of features. 140 141 y: array-like, shape = [n_samples] 142 Target values. 143 144 **kwargs: additional parameters to be passed to 145 self.cook_training_set or self.obj.fit 146 147 Returns: 148 149 self: object 150 151 """ 152 153 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 154 155 # if sample_weights, else: (must use self.row_index) 156 if sample_weight is not None: 157 self.obj.fit( 158 scaled_Z, 159 centered_y, 160 sample_weight=np.ravel(sample_weight, order="C")[ 161 self.index_row 162 ], 163 **kwargs 164 ) 165 166 return self 167 168 self.obj.fit(scaled_Z, centered_y, **kwargs) 169 170 return self 171 172 def predict(self, X, **kwargs): 173 """Predict test data X. 174 175 Parameters: 176 177 X: {array-like}, shape = [n_samples, n_features] 178 Training vectors, where n_samples is the number 179 of samples and n_features is the number of features. 180 181 **kwargs: additional parameters to be passed to 182 self.cook_test_set 183 184 Returns: 185 186 model predictions: {array-like} 187 188 """ 189 190 if len(X.shape) == 1: 191 n_features = X.shape[0] 192 new_X = mo.rbind( 193 X.reshape(1, n_features), 194 np.ones(n_features).reshape(1, n_features), 195 ) 196 197 return ( 198 self.y_mean_ 199 + self.obj.predict( 200 self.cook_test_set(new_X, **kwargs), **kwargs 201 ) 202 )[0] 203 204 return self.y_mean_ + self.obj.predict( 205 self.cook_test_set(X, **kwargs), **kwargs 206 ) 207 208 def score(self, X, y, scoring=None, **kwargs): 209 """ Score the model on test set features X and response y. 210 211 Parameters: 212 213 X: {array-like}, shape = [n_samples, n_features] 214 Training vectors, where n_samples is the number 215 of samples and n_features is the number of features 216 217 y: array-like, shape = [n_samples] 218 Target values 219 220 scoring: str 221 must be in ('explained_variance', 'neg_mean_absolute_error', \ 222 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 223 'neg_median_absolute_error', 'r2') 224 225 **kwargs: 226 additional parameters to be passed to scoring functions 227 228 Returns: 229 230 model scores: {array-like} 231 232 """ 233 234 preds = self.predict(X) 235 236 if type(preds) == tuple: # if there are std. devs in the predictions 237 preds = preds[0] 238 239 if scoring is None: 240 scoring = "neg_root_mean_squared_error" 241 242 # check inputs 243 assert scoring in ( 244 "explained_variance", 245 "neg_mean_absolute_error", 246 "neg_mean_squared_error", 247 "neg_mean_squared_log_error", 248 "neg_median_absolute_error", 249 "r2", 250 "neg_root_mean_squared_error", 251 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 252 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 253 'neg_median_absolute_error', 'r2', 'neg_root_mean_squared_error')" 254 255 scoring_options = { 256 "explained_variance": skm2.explained_variance_score, 257 "neg_mean_absolute_error": skm2.mean_absolute_error, 258 "neg_mean_squared_error": skm2.mean_squared_error, 259 "neg_mean_squared_log_error": skm2.mean_squared_log_error, 260 "neg_median_absolute_error": skm2.median_absolute_error, 261 "r2": skm2.r2_score, 262 "neg_root_mean_squared_error": partial( 263 skm2.mean_squared_error, squared=False 264 ), 265 } 266 267 return scoring_options[scoring](y, preds, **kwargs)
Custom Regression model
This class is used to 'augment' any regression model with transformed features.
Parameters:
obj: object
any object containing a method fit (obj.fit()) and a method predict
(obj.predict())
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not
(False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original predictors are included (True) in model's
fitting or not (False)
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
col_sample: float
percentage of covariates randomly chosen for training
row_sample: float
percentage of rows chosen for training, by stratified bootstrapping
seed: int
reproducibility seed for nodes_sim=='uniform'
type_fit: str
'regression'
backend: str
"cpu" or "gpu" or "tpu"
Examples:
TBD
92 def __init__( 93 self, 94 obj, 95 n_hidden_features=5, 96 activation_name="relu", 97 a=0.01, 98 nodes_sim="sobol", 99 bias=True, 100 dropout=0, 101 direct_link=True, 102 n_clusters=2, 103 cluster_encode=True, 104 type_clust="kmeans", 105 type_scaling=("std", "std", "std"), 106 col_sample=1, 107 row_sample=1, 108 seed=123, 109 backend="cpu", 110 ): 111 super().__init__( 112 obj=obj, 113 n_hidden_features=n_hidden_features, 114 activation_name=activation_name, 115 a=a, 116 nodes_sim=nodes_sim, 117 bias=bias, 118 dropout=dropout, 119 direct_link=direct_link, 120 n_clusters=n_clusters, 121 cluster_encode=cluster_encode, 122 type_clust=type_clust, 123 type_scaling=type_scaling, 124 col_sample=col_sample, 125 row_sample=row_sample, 126 seed=seed, 127 backend=backend, 128 ) 129 130 self.type_fit = "regression"
132 def fit(self, X, y, sample_weight=None, **kwargs): 133 """Fit custom model to training data (X, y). 134 135 Parameters: 136 137 X: {array-like}, shape = [n_samples, n_features] 138 Training vectors, where n_samples is the number 139 of samples and n_features is the number of features. 140 141 y: array-like, shape = [n_samples] 142 Target values. 143 144 **kwargs: additional parameters to be passed to 145 self.cook_training_set or self.obj.fit 146 147 Returns: 148 149 self: object 150 151 """ 152 153 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 154 155 # if sample_weights, else: (must use self.row_index) 156 if sample_weight is not None: 157 self.obj.fit( 158 scaled_Z, 159 centered_y, 160 sample_weight=np.ravel(sample_weight, order="C")[ 161 self.index_row 162 ], 163 **kwargs 164 ) 165 166 return self 167 168 self.obj.fit(scaled_Z, centered_y, **kwargs) 169 170 return self
Fit custom model to training data (X, y).
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
172 def predict(self, X, **kwargs): 173 """Predict test data X. 174 175 Parameters: 176 177 X: {array-like}, shape = [n_samples, n_features] 178 Training vectors, where n_samples is the number 179 of samples and n_features is the number of features. 180 181 **kwargs: additional parameters to be passed to 182 self.cook_test_set 183 184 Returns: 185 186 model predictions: {array-like} 187 188 """ 189 190 if len(X.shape) == 1: 191 n_features = X.shape[0] 192 new_X = mo.rbind( 193 X.reshape(1, n_features), 194 np.ones(n_features).reshape(1, n_features), 195 ) 196 197 return ( 198 self.y_mean_ 199 + self.obj.predict( 200 self.cook_test_set(new_X, **kwargs), **kwargs 201 ) 202 )[0] 203 204 return self.y_mean_ + self.obj.predict( 205 self.cook_test_set(X, **kwargs), **kwargs 206 )
Predict test data X.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
208 def score(self, X, y, scoring=None, **kwargs): 209 """ Score the model on test set features X and response y. 210 211 Parameters: 212 213 X: {array-like}, shape = [n_samples, n_features] 214 Training vectors, where n_samples is the number 215 of samples and n_features is the number of features 216 217 y: array-like, shape = [n_samples] 218 Target values 219 220 scoring: str 221 must be in ('explained_variance', 'neg_mean_absolute_error', \ 222 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 223 'neg_median_absolute_error', 'r2') 224 225 **kwargs: 226 additional parameters to be passed to scoring functions 227 228 Returns: 229 230 model scores: {array-like} 231 232 """ 233 234 preds = self.predict(X) 235 236 if type(preds) == tuple: # if there are std. devs in the predictions 237 preds = preds[0] 238 239 if scoring is None: 240 scoring = "neg_root_mean_squared_error" 241 242 # check inputs 243 assert scoring in ( 244 "explained_variance", 245 "neg_mean_absolute_error", 246 "neg_mean_squared_error", 247 "neg_mean_squared_log_error", 248 "neg_median_absolute_error", 249 "r2", 250 "neg_root_mean_squared_error", 251 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 252 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 253 'neg_median_absolute_error', 'r2', 'neg_root_mean_squared_error')" 254 255 scoring_options = { 256 "explained_variance": skm2.explained_variance_score, 257 "neg_mean_absolute_error": skm2.mean_absolute_error, 258 "neg_mean_squared_error": skm2.mean_squared_error, 259 "neg_mean_squared_log_error": skm2.mean_squared_log_error, 260 "neg_median_absolute_error": skm2.median_absolute_error, 261 "r2": skm2.r2_score, 262 "neg_root_mean_squared_error": partial( 263 skm2.mean_squared_error, squared=False 264 ), 265 } 266 267 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs:
additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.custom.custom.Custom
- obj
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
12class DeepClassifier(CustomClassifier, ClassifierMixin): 13 """ 14 Deep Classifier 15 16 Parameters 17 ---------- 18 verbose : int, optional (default=0) 19 Monitor progress when fitting. 20 21 Examples 22 -------- 23 >>> import nnetsauce as ns 24 >>> from sklearn.datasets import load_breast_cancer 25 >>> from sklearn.model_selection import train_test_split 26 >>> from sklearn.linear_model import LogisticRegressionCV 27 >>> data = load_breast_cancer() 28 >>> X = data.data 29 >>> y= data.target 30 >>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123) 31 >>> obj = LogisticRegressionCV() 32 >>> clf = ns.DeepClassifier(obj) 33 >>> clf.fit(X_train, y_train) 34 >>> print(clf.score(clf.predict(X_test), y_test)) 35 """ 36 37 def __init__( 38 self, 39 obj, 40 verbose=0, 41 # Defining depth 42 n_layers=3, 43 # CustomClassifier attributes 44 n_hidden_features=5, 45 activation_name="relu", 46 a=0.01, 47 nodes_sim="sobol", 48 bias=True, 49 dropout=0, 50 direct_link=True, 51 n_clusters=2, 52 cluster_encode=True, 53 type_clust="kmeans", 54 type_scaling=("std", "std", "std"), 55 col_sample=1, 56 row_sample=1, 57 seed=123, 58 backend="cpu", 59 ): 60 super().__init__( 61 obj=obj, 62 n_hidden_features=n_hidden_features, 63 activation_name=activation_name, 64 a=a, 65 nodes_sim=nodes_sim, 66 bias=bias, 67 dropout=dropout, 68 direct_link=direct_link, 69 n_clusters=n_clusters, 70 cluster_encode=cluster_encode, 71 type_clust=type_clust, 72 type_scaling=type_scaling, 73 col_sample=col_sample, 74 row_sample=row_sample, 75 seed=seed, 76 backend=backend, 77 ) 78 79 assert n_layers >= 2, "must have n_layers >= 2" 80 81 self.stacked_obj = obj 82 self.verbose = verbose 83 self.n_layers = n_layers 84 85 def fit(self, X, y): 86 """Fit Classification algorithms to X and y. 87 Parameters 88 ---------- 89 X : array-like, 90 Training vectors, where rows is the number of samples 91 and columns is the number of features. 92 y : array-like, 93 Training vectors, where rows is the number of samples 94 and columns is the number of features. 95 Returns 96 ------- 97 A fitted object 98 """ 99 100 if isinstance(X, np.ndarray): 101 X = pd.DataFrame(X) 102 103 # init layer 104 self.stacked_obj = CustomClassifier( 105 obj=self.stacked_obj, 106 n_hidden_features=self.n_hidden_features, 107 activation_name=self.activation_name, 108 a=self.a, 109 nodes_sim=self.nodes_sim, 110 bias=self.bias, 111 dropout=self.dropout, 112 direct_link=self.direct_link, 113 n_clusters=self.n_clusters, 114 cluster_encode=self.cluster_encode, 115 type_clust=self.type_clust, 116 type_scaling=self.type_scaling, 117 col_sample=self.col_sample, 118 row_sample=self.row_sample, 119 seed=self.seed, 120 backend=self.backend, 121 ) 122 123 self.stacked_obj.fit(X, y) 124 125 if self.verbose > 0: 126 iterator = tqdm(range(self.n_layers - 1)) 127 else: 128 iterator = range(self.n_layers - 1) 129 130 for _ in iterator: 131 self.stacked_obj = deepcopy( 132 CustomClassifier( 133 obj=self.stacked_obj, 134 n_hidden_features=self.n_hidden_features, 135 activation_name=self.activation_name, 136 a=self.a, 137 nodes_sim=self.nodes_sim, 138 bias=self.bias, 139 dropout=self.dropout, 140 direct_link=self.direct_link, 141 n_clusters=self.n_clusters, 142 cluster_encode=self.cluster_encode, 143 type_clust=self.type_clust, 144 type_scaling=self.type_scaling, 145 col_sample=self.col_sample, 146 row_sample=self.row_sample, 147 seed=self.seed, 148 backend=self.backend, 149 ) 150 ) 151 152 # self.stacked_obj.fit(X, y) 153 154 self.stacked_obj.fit(X, y) 155 156 self.obj = deepcopy(self.stacked_obj) 157 158 return self.obj 159 160 def predict(self, X): 161 return self.obj.predict(X) 162 163 def predict_proba(self, X): 164 return self.obj.predict_proba(X) 165 166 def score(self, X, y, scoring=None): 167 return self.obj.score(X, y, scoring)
Deep Classifier
Parameters
verbose : int, optional (default=0) Monitor progress when fitting.
Examples
>>> import nnetsauce as ns
>>> from sklearn.datasets import load_breast_cancer
>>> from sklearn.model_selection import train_test_split
>>> from sklearn.linear_model import LogisticRegressionCV
>>> data = load_breast_cancer()
>>> X = data.data
>>> y= data.target
>>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
>>> obj = LogisticRegressionCV()
>>> clf = ns.DeepClassifier(obj)
>>> clf.fit(X_train, y_train)
>>> print(clf.score(clf.predict(X_test), y_test))
37 def __init__( 38 self, 39 obj, 40 verbose=0, 41 # Defining depth 42 n_layers=3, 43 # CustomClassifier attributes 44 n_hidden_features=5, 45 activation_name="relu", 46 a=0.01, 47 nodes_sim="sobol", 48 bias=True, 49 dropout=0, 50 direct_link=True, 51 n_clusters=2, 52 cluster_encode=True, 53 type_clust="kmeans", 54 type_scaling=("std", "std", "std"), 55 col_sample=1, 56 row_sample=1, 57 seed=123, 58 backend="cpu", 59 ): 60 super().__init__( 61 obj=obj, 62 n_hidden_features=n_hidden_features, 63 activation_name=activation_name, 64 a=a, 65 nodes_sim=nodes_sim, 66 bias=bias, 67 dropout=dropout, 68 direct_link=direct_link, 69 n_clusters=n_clusters, 70 cluster_encode=cluster_encode, 71 type_clust=type_clust, 72 type_scaling=type_scaling, 73 col_sample=col_sample, 74 row_sample=row_sample, 75 seed=seed, 76 backend=backend, 77 ) 78 79 assert n_layers >= 2, "must have n_layers >= 2" 80 81 self.stacked_obj = obj 82 self.verbose = verbose 83 self.n_layers = n_layers
85 def fit(self, X, y): 86 """Fit Classification algorithms to X and y. 87 Parameters 88 ---------- 89 X : array-like, 90 Training vectors, where rows is the number of samples 91 and columns is the number of features. 92 y : array-like, 93 Training vectors, where rows is the number of samples 94 and columns is the number of features. 95 Returns 96 ------- 97 A fitted object 98 """ 99 100 if isinstance(X, np.ndarray): 101 X = pd.DataFrame(X) 102 103 # init layer 104 self.stacked_obj = CustomClassifier( 105 obj=self.stacked_obj, 106 n_hidden_features=self.n_hidden_features, 107 activation_name=self.activation_name, 108 a=self.a, 109 nodes_sim=self.nodes_sim, 110 bias=self.bias, 111 dropout=self.dropout, 112 direct_link=self.direct_link, 113 n_clusters=self.n_clusters, 114 cluster_encode=self.cluster_encode, 115 type_clust=self.type_clust, 116 type_scaling=self.type_scaling, 117 col_sample=self.col_sample, 118 row_sample=self.row_sample, 119 seed=self.seed, 120 backend=self.backend, 121 ) 122 123 self.stacked_obj.fit(X, y) 124 125 if self.verbose > 0: 126 iterator = tqdm(range(self.n_layers - 1)) 127 else: 128 iterator = range(self.n_layers - 1) 129 130 for _ in iterator: 131 self.stacked_obj = deepcopy( 132 CustomClassifier( 133 obj=self.stacked_obj, 134 n_hidden_features=self.n_hidden_features, 135 activation_name=self.activation_name, 136 a=self.a, 137 nodes_sim=self.nodes_sim, 138 bias=self.bias, 139 dropout=self.dropout, 140 direct_link=self.direct_link, 141 n_clusters=self.n_clusters, 142 cluster_encode=self.cluster_encode, 143 type_clust=self.type_clust, 144 type_scaling=self.type_scaling, 145 col_sample=self.col_sample, 146 row_sample=self.row_sample, 147 seed=self.seed, 148 backend=self.backend, 149 ) 150 ) 151 152 # self.stacked_obj.fit(X, y) 153 154 self.stacked_obj.fit(X, y) 155 156 self.obj = deepcopy(self.stacked_obj) 157 158 return self.obj
Fit Classification algorithms to X and y.
Parameters
X : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y : array-like, Training vectors, where rows is the number of samples and columns is the number of features.
Returns
A fitted object
Predict test data X.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
Predict probabilities for test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
probability estimates for test data: {array-like}
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs:
additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.custom.custom.Custom
- obj
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
12class DeepRegressor(CustomRegressor, RegressorMixin): 13 """ 14 Deep Regressor 15 16 Parameters 17 ---------- 18 verbose : int, optional (default=0) 19 Monitor progress when fitting. 20 21 Examples 22 -------- 23 >>> import nnetsauce as ns 24 >>> from sklearn.datasets import load_diabetes 25 >>> from sklearn.model_selection import train_test_split 26 >>> from sklearn.linear_model import RidgeCV 27 >>> data = load_diabetes() 28 >>> X = data.data 29 >>> y= data.target 30 >>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123) 31 >>> obj = RidgeCV() 32 >>> clf = ns.DeepRegressor(obj) 33 >>> clf.fit(X_train, y_train) 34 >>> print(clf.score(clf.predict(X_test), y_test)) 35 """ 36 37 def __init__( 38 self, 39 obj, 40 verbose=0, 41 # Defining depth 42 n_layers=3, 43 # CustomRegressor attributes 44 n_hidden_features=5, 45 activation_name="relu", 46 a=0.01, 47 nodes_sim="sobol", 48 bias=True, 49 dropout=0, 50 direct_link=True, 51 n_clusters=2, 52 cluster_encode=True, 53 type_clust="kmeans", 54 type_scaling=("std", "std", "std"), 55 col_sample=1, 56 row_sample=1, 57 seed=123, 58 backend="cpu", 59 ): 60 super().__init__( 61 obj=obj, 62 n_hidden_features=n_hidden_features, 63 activation_name=activation_name, 64 a=a, 65 nodes_sim=nodes_sim, 66 bias=bias, 67 dropout=dropout, 68 direct_link=direct_link, 69 n_clusters=n_clusters, 70 cluster_encode=cluster_encode, 71 type_clust=type_clust, 72 type_scaling=type_scaling, 73 col_sample=col_sample, 74 row_sample=row_sample, 75 seed=seed, 76 backend=backend, 77 ) 78 79 assert n_layers >= 2, "must have n_layers >= 2" 80 81 self.stacked_obj = obj 82 self.verbose = verbose 83 self.n_layers = n_layers 84 85 def fit(self, X, y): 86 """Fit Regression algorithms to X and y. 87 Parameters 88 ---------- 89 X : array-like, 90 Training vectors, where rows is the number of samples 91 and columns is the number of features. 92 y : array-like, 93 Training vectors, where rows is the number of samples 94 and columns is the number of features. 95 Returns 96 ------- 97 A fitted object 98 """ 99 100 if isinstance(X, np.ndarray): 101 X = pd.DataFrame(X) 102 103 # init layer 104 self.stacked_obj = CustomRegressor( 105 obj=self.stacked_obj, 106 n_hidden_features=self.n_hidden_features, 107 activation_name=self.activation_name, 108 a=self.a, 109 nodes_sim=self.nodes_sim, 110 bias=self.bias, 111 dropout=self.dropout, 112 direct_link=self.direct_link, 113 n_clusters=self.n_clusters, 114 cluster_encode=self.cluster_encode, 115 type_clust=self.type_clust, 116 type_scaling=self.type_scaling, 117 col_sample=self.col_sample, 118 row_sample=self.row_sample, 119 seed=self.seed, 120 backend=self.backend, 121 ) 122 123 # self.stacked_obj.fit(X, y) 124 125 if self.verbose > 0: 126 iterator = tqdm(range(self.n_layers - 1)) 127 else: 128 iterator = range(self.n_layers - 1) 129 130 for _ in iterator: 131 self.stacked_obj = deepcopy( 132 CustomRegressor( 133 obj=self.stacked_obj, 134 n_hidden_features=self.n_hidden_features, 135 activation_name=self.activation_name, 136 a=self.a, 137 nodes_sim=self.nodes_sim, 138 bias=self.bias, 139 dropout=self.dropout, 140 direct_link=self.direct_link, 141 n_clusters=self.n_clusters, 142 cluster_encode=self.cluster_encode, 143 type_clust=self.type_clust, 144 type_scaling=self.type_scaling, 145 col_sample=self.col_sample, 146 row_sample=self.row_sample, 147 seed=self.seed, 148 backend=self.backend, 149 ) 150 ) 151 152 # self.stacked_obj.fit(X, y) 153 154 self.stacked_obj.fit(X, y) 155 156 self.obj = deepcopy(self.stacked_obj) 157 158 return self.obj 159 160 def predict(self, X): 161 return self.obj.predict(X) 162 163 def score(self, X, y, scoring=None): 164 return self.obj.score(X, y, scoring)
Deep Regressor
Parameters
verbose : int, optional (default=0) Monitor progress when fitting.
Examples
>>> import nnetsauce as ns
>>> from sklearn.datasets import load_diabetes
>>> from sklearn.model_selection import train_test_split
>>> from sklearn.linear_model import RidgeCV
>>> data = load_diabetes()
>>> X = data.data
>>> y= data.target
>>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
>>> obj = RidgeCV()
>>> clf = ns.DeepRegressor(obj)
>>> clf.fit(X_train, y_train)
>>> print(clf.score(clf.predict(X_test), y_test))
37 def __init__( 38 self, 39 obj, 40 verbose=0, 41 # Defining depth 42 n_layers=3, 43 # CustomRegressor attributes 44 n_hidden_features=5, 45 activation_name="relu", 46 a=0.01, 47 nodes_sim="sobol", 48 bias=True, 49 dropout=0, 50 direct_link=True, 51 n_clusters=2, 52 cluster_encode=True, 53 type_clust="kmeans", 54 type_scaling=("std", "std", "std"), 55 col_sample=1, 56 row_sample=1, 57 seed=123, 58 backend="cpu", 59 ): 60 super().__init__( 61 obj=obj, 62 n_hidden_features=n_hidden_features, 63 activation_name=activation_name, 64 a=a, 65 nodes_sim=nodes_sim, 66 bias=bias, 67 dropout=dropout, 68 direct_link=direct_link, 69 n_clusters=n_clusters, 70 cluster_encode=cluster_encode, 71 type_clust=type_clust, 72 type_scaling=type_scaling, 73 col_sample=col_sample, 74 row_sample=row_sample, 75 seed=seed, 76 backend=backend, 77 ) 78 79 assert n_layers >= 2, "must have n_layers >= 2" 80 81 self.stacked_obj = obj 82 self.verbose = verbose 83 self.n_layers = n_layers
85 def fit(self, X, y): 86 """Fit Regression algorithms to X and y. 87 Parameters 88 ---------- 89 X : array-like, 90 Training vectors, where rows is the number of samples 91 and columns is the number of features. 92 y : array-like, 93 Training vectors, where rows is the number of samples 94 and columns is the number of features. 95 Returns 96 ------- 97 A fitted object 98 """ 99 100 if isinstance(X, np.ndarray): 101 X = pd.DataFrame(X) 102 103 # init layer 104 self.stacked_obj = CustomRegressor( 105 obj=self.stacked_obj, 106 n_hidden_features=self.n_hidden_features, 107 activation_name=self.activation_name, 108 a=self.a, 109 nodes_sim=self.nodes_sim, 110 bias=self.bias, 111 dropout=self.dropout, 112 direct_link=self.direct_link, 113 n_clusters=self.n_clusters, 114 cluster_encode=self.cluster_encode, 115 type_clust=self.type_clust, 116 type_scaling=self.type_scaling, 117 col_sample=self.col_sample, 118 row_sample=self.row_sample, 119 seed=self.seed, 120 backend=self.backend, 121 ) 122 123 # self.stacked_obj.fit(X, y) 124 125 if self.verbose > 0: 126 iterator = tqdm(range(self.n_layers - 1)) 127 else: 128 iterator = range(self.n_layers - 1) 129 130 for _ in iterator: 131 self.stacked_obj = deepcopy( 132 CustomRegressor( 133 obj=self.stacked_obj, 134 n_hidden_features=self.n_hidden_features, 135 activation_name=self.activation_name, 136 a=self.a, 137 nodes_sim=self.nodes_sim, 138 bias=self.bias, 139 dropout=self.dropout, 140 direct_link=self.direct_link, 141 n_clusters=self.n_clusters, 142 cluster_encode=self.cluster_encode, 143 type_clust=self.type_clust, 144 type_scaling=self.type_scaling, 145 col_sample=self.col_sample, 146 row_sample=self.row_sample, 147 seed=self.seed, 148 backend=self.backend, 149 ) 150 ) 151 152 # self.stacked_obj.fit(X, y) 153 154 self.stacked_obj.fit(X, y) 155 156 self.obj = deepcopy(self.stacked_obj) 157 158 return self.obj
Fit Regression algorithms to X and y.
Parameters
X : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y : array-like, Training vectors, where rows is the number of samples and columns is the number of features.
Returns
A fitted object
Predict test data X.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
Score the model on test set features X and response y.
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs:
additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.custom.custom.Custom
- obj
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
10class DeepMTS(MTS): 11 """Univariate and multivariate time series (DeepMTS) forecasting with Quasi-Randomized networks (Work in progress /!\) 12 13 Parameters: 14 15 obj: object. 16 any object containing a method fit (obj.fit()) and a method predict 17 (obj.predict()). 18 19 n_layers: int. 20 number of layers in the neural network. 21 22 n_hidden_features: int. 23 number of nodes in the hidden layer. 24 25 activation_name: str. 26 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'. 27 28 a: float. 29 hyperparameter for 'prelu' or 'elu' activation function. 30 31 nodes_sim: str. 32 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 33 'uniform'. 34 35 bias: boolean. 36 indicates if the hidden layer contains a bias term (True) or not 37 (False). 38 39 dropout: float. 40 regularization parameter; (random) percentage of nodes dropped out 41 of the training. 42 43 direct_link: boolean. 44 indicates if the original predictors are included (True) in model's fitting or not (False). 45 46 n_clusters: int. 47 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering). 48 49 cluster_encode: bool. 50 defines how the variable containing clusters is treated (default is one-hot) 51 if `False`, then labels are used, without one-hot encoding. 52 53 type_clust: str. 54 type of clustering method: currently k-means ('kmeans') or Gaussian 55 Mixture Model ('gmm'). 56 57 type_scaling: a tuple of 3 strings. 58 scaling methods for inputs, hidden layer, and clustering respectively 59 (and when relevant). 60 Currently available: standardization ('std') or MinMax scaling ('minmax'). 61 62 lags: int. 63 number of lags used for each time series. 64 65 replications: int. 66 number of replications (if needed, for predictive simulation). Default is 'None'. 67 68 kernel: str. 69 the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'. 70 71 agg: str. 72 either "mean" or "median" for simulation of bootstrap aggregating 73 74 seed: int. 75 reproducibility seed for nodes_sim=='uniform' or predictive simulation. 76 77 backend: str. 78 "cpu" or "gpu" or "tpu". 79 80 verbose: int. 81 0: not printing; 1: printing 82 83 show_progress: bool. 84 True: progress bar when fitting each series; False: no progress bar when fitting each series 85 86 Attributes: 87 88 fit_objs_: dict 89 objects adjusted to each individual time series 90 91 y_: {array-like} 92 DeepMTS responses (most recent observations first) 93 94 X_: {array-like} 95 DeepMTS lags 96 97 xreg_: {array-like} 98 external regressors 99 100 y_means_: dict 101 a dictionary of each series mean values 102 103 preds_: {array-like} 104 successive model predictions 105 106 preds_std_: {array-like} 107 standard deviation around the predictions 108 109 return_std_: boolean 110 return uncertainty or not (set in predict) 111 112 df_: data frame 113 the input data frame, in case a data.frame is provided to `fit` 114 115 Examples: 116 117 Example 1: 118 119 ```python 120 import nnetsauce as ns 121 import numpy as np 122 from sklearn import linear_model 123 np.random.seed(123) 124 125 M = np.random.rand(10, 3) 126 M[:,0] = 10*M[:,0] 127 M[:,2] = 25*M[:,2] 128 print(M) 129 130 # Adjust Bayesian Ridge 131 regr4 = linear_model.BayesianRidge() 132 obj_DeepMTS = ns.DeepMTS(regr4, lags = 1, n_hidden_features=5) 133 obj_DeepMTS.fit(M) 134 print(obj_DeepMTS.predict()) 135 136 # with credible intervals 137 print(obj_DeepMTS.predict(return_std=True, level=80)) 138 139 print(obj_DeepMTS.predict(return_std=True, level=95)) 140 ``` 141 142 Example 2: 143 144 ```python 145 import nnetsauce as ns 146 import numpy as np 147 from sklearn import linear_model 148 149 dataset = { 150 'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'], 151 'series1' : [34, 30, 35.6, 33.3, 38.1], 152 'series2' : [4, 5.5, 5.6, 6.3, 5.1], 153 'series3' : [100, 100.5, 100.6, 100.2, 100.1]} 154 df = pd.DataFrame(dataset).set_index('date') 155 print(df) 156 157 # Adjust Bayesian Ridge 158 regr5 = linear_model.BayesianRidge() 159 obj_DeepMTS = ns.DeepMTS(regr5, lags = 1, n_hidden_features=5) 160 obj_DeepMTS.fit(df) 161 print(obj_DeepMTS.predict()) 162 163 # with credible intervals 164 print(obj_DeepMTS.predict(return_std=True, level=80)) 165 166 print(obj_DeepMTS.predict(return_std=True, level=95)) 167 ``` 168 """ 169 170 # construct the object ----- 171 172 def __init__( 173 self, 174 obj, 175 n_layers=3, 176 n_hidden_features=5, 177 activation_name="relu", 178 a=0.01, 179 nodes_sim="sobol", 180 bias=True, 181 dropout=0, 182 direct_link=True, 183 n_clusters=2, 184 cluster_encode=True, 185 type_clust="kmeans", 186 type_scaling=("std", "std", "std"), 187 lags=1, 188 type_pi="kde", 189 replications=None, 190 kernel=None, 191 agg="mean", 192 seed=123, 193 backend="cpu", 194 verbose=0, 195 show_progress=True, 196 ): 197 assert int(lags) == lags, "parameter 'lags' should be an integer" 198 assert n_layers >= 2, "must have n_layers >= 2" 199 self.n_layers = int(n_layers) 200 201 self.obj = DeepRegressor( 202 obj=obj, 203 verbose=0, 204 n_layers=self.n_layers, 205 n_hidden_features=n_hidden_features, 206 activation_name=activation_name, 207 a=a, 208 nodes_sim=nodes_sim, 209 bias=bias, 210 dropout=dropout, 211 direct_link=direct_link, 212 n_clusters=n_clusters, 213 cluster_encode=cluster_encode, 214 type_clust=type_clust, 215 type_scaling=type_scaling, 216 seed=seed, 217 backend=backend, 218 ) 219 220 super().__init__( 221 obj=self.obj, 222 n_hidden_features=n_hidden_features, 223 activation_name=activation_name, 224 a=a, 225 nodes_sim=nodes_sim, 226 bias=bias, 227 dropout=dropout, 228 direct_link=direct_link, 229 n_clusters=n_clusters, 230 cluster_encode=cluster_encode, 231 type_clust=type_clust, 232 type_scaling=type_scaling, 233 seed=seed, 234 type_pi=type_pi, 235 replications=replications, 236 kernel=kernel, 237 agg=agg, 238 backend=backend, 239 verbose=verbose, 240 show_progress=show_progress, 241 )
Univariate and multivariate time series (DeepMTS) forecasting with Quasi-Randomized networks (Work in progress /!)
Parameters:
obj: object.
any object containing a method fit (obj.fit()) and a method predict
(obj.predict()).
n_layers: int.
number of layers in the neural network.
n_hidden_features: int.
number of nodes in the hidden layer.
activation_name: str.
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'.
a: float.
hyperparameter for 'prelu' or 'elu' activation function.
nodes_sim: str.
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'.
bias: boolean.
indicates if the hidden layer contains a bias term (True) or not
(False).
dropout: float.
regularization parameter; (random) percentage of nodes dropped out
of the training.
direct_link: boolean.
indicates if the original predictors are included (True) in model's fitting or not (False).
n_clusters: int.
number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering).
cluster_encode: bool.
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding.
type_clust: str.
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm').
type_scaling: a tuple of 3 strings.
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax').
lags: int.
number of lags used for each time series.
replications: int.
number of replications (if needed, for predictive simulation). Default is 'None'.
kernel: str.
the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'.
agg: str.
either "mean" or "median" for simulation of bootstrap aggregating
seed: int.
reproducibility seed for nodes_sim=='uniform' or predictive simulation.
backend: str.
"cpu" or "gpu" or "tpu".
verbose: int.
0: not printing; 1: printing
show_progress: bool.
True: progress bar when fitting each series; False: no progress bar when fitting each series
Attributes:
fit_objs_: dict
objects adjusted to each individual time series
y_: {array-like}
DeepMTS responses (most recent observations first)
X_: {array-like}
DeepMTS lags
xreg_: {array-like}
external regressors
y_means_: dict
a dictionary of each series mean values
preds_: {array-like}
successive model predictions
preds_std_: {array-like}
standard deviation around the predictions
return_std_: boolean
return uncertainty or not (set in predict)
df_: data frame
the input data frame, in case a data.frame is provided to `fit`
Examples:
Example 1:
import nnetsauce as ns
import numpy as np
from sklearn import linear_model
np.random.seed(123)
M = np.random.rand(10, 3)
M[:,0] = 10*M[:,0]
M[:,2] = 25*M[:,2]
print(M)
# Adjust Bayesian Ridge
regr4 = linear_model.BayesianRidge()
obj_DeepMTS = ns.DeepMTS(regr4, lags = 1, n_hidden_features=5)
obj_DeepMTS.fit(M)
print(obj_DeepMTS.predict())
# with credible intervals
print(obj_DeepMTS.predict(return_std=True, level=80))
print(obj_DeepMTS.predict(return_std=True, level=95))
Example 2:
import nnetsauce as ns
import numpy as np
from sklearn import linear_model
dataset = {
'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'],
'series1' : [34, 30, 35.6, 33.3, 38.1],
'series2' : [4, 5.5, 5.6, 6.3, 5.1],
'series3' : [100, 100.5, 100.6, 100.2, 100.1]}
df = pd.DataFrame(dataset).set_index('date')
print(df)
# Adjust Bayesian Ridge
regr5 = linear_model.BayesianRidge()
obj_DeepMTS = ns.DeepMTS(regr5, lags = 1, n_hidden_features=5)
obj_DeepMTS.fit(df)
print(obj_DeepMTS.predict())
# with credible intervals
print(obj_DeepMTS.predict(return_std=True, level=80))
print(obj_DeepMTS.predict(return_std=True, level=95))
172 def __init__( 173 self, 174 obj, 175 n_layers=3, 176 n_hidden_features=5, 177 activation_name="relu", 178 a=0.01, 179 nodes_sim="sobol", 180 bias=True, 181 dropout=0, 182 direct_link=True, 183 n_clusters=2, 184 cluster_encode=True, 185 type_clust="kmeans", 186 type_scaling=("std", "std", "std"), 187 lags=1, 188 type_pi="kde", 189 replications=None, 190 kernel=None, 191 agg="mean", 192 seed=123, 193 backend="cpu", 194 verbose=0, 195 show_progress=True, 196 ): 197 assert int(lags) == lags, "parameter 'lags' should be an integer" 198 assert n_layers >= 2, "must have n_layers >= 2" 199 self.n_layers = int(n_layers) 200 201 self.obj = DeepRegressor( 202 obj=obj, 203 verbose=0, 204 n_layers=self.n_layers, 205 n_hidden_features=n_hidden_features, 206 activation_name=activation_name, 207 a=a, 208 nodes_sim=nodes_sim, 209 bias=bias, 210 dropout=dropout, 211 direct_link=direct_link, 212 n_clusters=n_clusters, 213 cluster_encode=cluster_encode, 214 type_clust=type_clust, 215 type_scaling=type_scaling, 216 seed=seed, 217 backend=backend, 218 ) 219 220 super().__init__( 221 obj=self.obj, 222 n_hidden_features=n_hidden_features, 223 activation_name=activation_name, 224 a=a, 225 nodes_sim=nodes_sim, 226 bias=bias, 227 dropout=dropout, 228 direct_link=direct_link, 229 n_clusters=n_clusters, 230 cluster_encode=cluster_encode, 231 type_clust=type_clust, 232 type_scaling=type_scaling, 233 seed=seed, 234 type_pi=type_pi, 235 replications=replications, 236 kernel=kernel, 237 agg=agg, 238 backend=backend, 239 verbose=verbose, 240 show_progress=show_progress, 241 )
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- MTS
- n_series
- lags
- type_pi
- replications
- kernel
- agg
- verbose
- show_progress
- series_names
- input_dates
- fit_objs_
- y_
- X_
- xreg_
- y_means_
- mean_
- upper_
- lower_
- output_dates_
- preds_std_
- alpha_
- return_std_
- df_
- residuals_
- residuals_sims_
- kde_
- sims_
- fit
- predict
- score
- plot
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
16class GLMClassifier(GLM, ClassifierMixin): 17 """Generalized 'linear' models using quasi-randomized networks (classification) 18 19 Parameters: 20 21 n_hidden_features: int 22 number of nodes in the hidden layer 23 24 lambda1: float 25 regularization parameter for GLM coefficients on original features 26 27 alpha1: float 28 controls compromize between l1 and l2 norm of GLM coefficients on original features 29 30 lambda2: float 31 regularization parameter for GLM coefficients on nonlinear features 32 33 alpha2: float 34 controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features 35 36 activation_name: str 37 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 38 39 a: float 40 hyperparameter for 'prelu' or 'elu' activation function 41 42 nodes_sim: str 43 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 44 'uniform' 45 46 bias: boolean 47 indicates if the hidden layer contains a bias term (True) or not 48 (False) 49 50 dropout: float 51 regularization parameter; (random) percentage of nodes dropped out 52 of the training 53 54 direct_link: boolean 55 indicates if the original predictors are included (True) in model's 56 fitting or not (False) 57 58 n_clusters: int 59 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: 60 no clustering) 61 62 cluster_encode: bool 63 defines how the variable containing clusters is treated (default is one-hot) 64 if `False`, then labels are used, without one-hot encoding 65 66 type_clust: str 67 type of clustering method: currently k-means ('kmeans') or Gaussian 68 Mixture Model ('gmm') 69 70 type_scaling: a tuple of 3 strings 71 scaling methods for inputs, hidden layer, and clustering respectively 72 (and when relevant). 73 Currently available: standardization ('std') or MinMax scaling ('minmax') 74 75 optimizer: object 76 optimizer, from class nnetsauce.Optimizer 77 78 seed: int 79 reproducibility seed for nodes_sim=='uniform' 80 81 Attributes: 82 83 beta_: vector 84 regression coefficients 85 86 Examples: 87 88 See [https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_classification.py) 89 90 """ 91 92 # construct the object ----- 93 94 def __init__( 95 self, 96 n_hidden_features=5, 97 lambda1=0.01, 98 alpha1=0.5, 99 lambda2=0.01, 100 alpha2=0.5, 101 family="expit", 102 activation_name="relu", 103 a=0.01, 104 nodes_sim="sobol", 105 bias=True, 106 dropout=0, 107 direct_link=True, 108 n_clusters=2, 109 cluster_encode=True, 110 type_clust="kmeans", 111 type_scaling=("std", "std", "std"), 112 optimizer=Optimizer(), 113 seed=123, 114 ): 115 super().__init__( 116 n_hidden_features=n_hidden_features, 117 lambda1=lambda1, 118 alpha1=alpha1, 119 lambda2=lambda2, 120 alpha2=alpha2, 121 activation_name=activation_name, 122 a=a, 123 nodes_sim=nodes_sim, 124 bias=bias, 125 dropout=dropout, 126 direct_link=direct_link, 127 n_clusters=n_clusters, 128 cluster_encode=cluster_encode, 129 type_clust=type_clust, 130 type_scaling=type_scaling, 131 optimizer=optimizer, 132 seed=seed, 133 ) 134 135 self.family = family 136 137 def logit_loss(self, Y, row_index, XB): 138 self.n_classes = Y.shape[1] # len(np.unique(y)) 139 # Y = mo.one_hot_encode2(y, self.n_classes) 140 # Y = self.optimizer.one_hot_encode(y, self.n_classes) 141 142 # max_double = 709.0 # only if softmax 143 # XB[XB > max_double] = max_double 144 XB[XB > 709.0] = 709.0 145 146 if row_index is None: 147 return -np.mean(np.sum(Y * XB, axis=1) - logsumexp(XB)) 148 149 return -np.mean(np.sum(Y[row_index, :] * XB, axis=1) - logsumexp(XB)) 150 151 def expit_erf_loss(self, Y, row_index, XB): 152 # self.n_classes = len(np.unique(y)) 153 # Y = mo.one_hot_encode2(y, self.n_classes) 154 # Y = self.optimizer.one_hot_encode(y, self.n_classes) 155 self.n_classes = Y.shape[1] 156 157 if row_index is None: 158 return -np.mean(np.sum(Y * XB, axis=1) - logsumexp(XB)) 159 160 return -np.mean(np.sum(Y[row_index, :] * XB, axis=1) - logsumexp(XB)) 161 162 def loss_func( 163 self, 164 beta, 165 group_index, 166 X, 167 Y, 168 y, 169 row_index=None, 170 type_loss="logit", 171 **kwargs 172 ): 173 res = { 174 "logit": self.logit_loss, 175 "expit": self.expit_erf_loss, 176 "erf": self.expit_erf_loss, 177 } 178 179 if row_index is None: 180 row_index = range(len(y)) 181 XB = self.compute_XB( 182 X, 183 beta=np.reshape(beta, (X.shape[1], self.n_classes), order="F"), 184 ) 185 186 return res[type_loss](Y, row_index, XB) + self.compute_penalty( 187 group_index=group_index, beta=beta 188 ) 189 190 XB = self.compute_XB( 191 X, 192 beta=np.reshape(beta, (X.shape[1], self.n_classes), order="F"), 193 row_index=row_index, 194 ) 195 196 return res[type_loss](Y, row_index, XB) + self.compute_penalty( 197 group_index=group_index, beta=beta 198 ) 199 200 def fit(self, X, y, **kwargs): 201 """Fit GLM model to training data (X, y). 202 203 Args: 204 205 X: {array-like}, shape = [n_samples, n_features] 206 Training vectors, where n_samples is the number 207 of samples and n_features is the number of features. 208 209 y: array-like, shape = [n_samples] 210 Target values. 211 212 **kwargs: additional parameters to be passed to 213 self.cook_training_set or self.obj.fit 214 215 Returns: 216 217 self: object 218 219 """ 220 221 assert mx.is_factor( 222 y 223 ), "y must contain only integers" # change is_factor and subsampling everywhere 224 225 self.beta_ = None 226 227 n, p = X.shape 228 229 self.group_index = n * X.shape[1] 230 231 self.n_classes = len(np.unique(y)) 232 233 output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 234 235 # Y = mo.one_hot_encode2(output_y, self.n_classes) 236 Y = self.optimizer.one_hot_encode(output_y, self.n_classes) 237 238 # initialization 239 beta_ = np.linalg.lstsq(scaled_Z, Y, rcond=None)[0] 240 241 # optimization 242 # fit(self, loss_func, response, x0, **kwargs): 243 # loss_func(self, beta, group_index, X, y, 244 # row_index=None, type_loss="gaussian", 245 # **kwargs) 246 self.optimizer.fit( 247 self.loss_func, 248 response=y, 249 x0=beta_.flatten(order="F"), 250 group_index=self.group_index, 251 X=scaled_Z, 252 Y=Y, 253 y=y, 254 type_loss=self.family, 255 ) 256 257 self.beta_ = self.optimizer.results[0] 258 259 return self 260 261 def predict(self, X, **kwargs): 262 """Predict test data X. 263 264 Args: 265 266 X: {array-like}, shape = [n_samples, n_features] 267 Training vectors, where n_samples is the number 268 of samples and n_features is the number of features. 269 270 **kwargs: additional parameters to be passed to 271 self.cook_test_set 272 273 Returns: 274 275 model predictions: {array-like} 276 277 """ 278 279 return np.argmax(self.predict_proba(X, **kwargs), axis=1) 280 281 def predict_proba(self, X, **kwargs): 282 """Predict probabilities for test data X. 283 284 Args: 285 286 X: {array-like}, shape = [n_samples, n_features] 287 Training vectors, where n_samples is the number 288 of samples and n_features is the number of features. 289 290 **kwargs: additional parameters to be passed to 291 self.cook_test_set 292 293 Returns: 294 295 probability estimates for test data: {array-like} 296 297 """ 298 if len(X.shape) == 1: 299 n_features = X.shape[0] 300 new_X = mo.rbind( 301 X.reshape(1, n_features), 302 np.ones(n_features).reshape(1, n_features), 303 ) 304 305 Z = self.cook_test_set(new_X, **kwargs) 306 307 else: 308 Z = self.cook_test_set(X, **kwargs) 309 310 ZB = mo.safe_sparse_dot( 311 Z, 312 self.beta_.reshape( 313 self.n_classes, 314 X.shape[1] + self.n_hidden_features + self.n_clusters, 315 ).T, 316 ) 317 318 if self.family == "logit": 319 exp_ZB = np.exp(ZB) 320 321 return exp_ZB / exp_ZB.sum(axis=1)[:, None] 322 323 if self.family == "expit": 324 exp_ZB = expit(ZB) 325 326 return exp_ZB / exp_ZB.sum(axis=1)[:, None] 327 328 if self.family == "erf": 329 exp_ZB = 0.5 * (1 + erf(ZB)) 330 331 return exp_ZB / exp_ZB.sum(axis=1)[:, None] 332 333 def score(self, X, y, scoring=None, **kwargs): 334 """ Score the model on test set features X and response y. 335 336 Args: 337 338 X: {array-like}, shape = [n_samples, n_features] 339 Training vectors, where n_samples is the number 340 of samples and n_features is the number of features 341 342 y: array-like, shape = [n_samples] 343 Target values 344 345 scoring: str 346 must be in ('accuracy', 'average_precision', \ 347 'brier_score_loss', 'f1', 'f1_micro', \ 348 'f1_macro', 'f1_weighted', 'f1_samples', \ 349 'neg_log_loss', 'precision', 'recall', \ 350 'roc_auc') 351 352 **kwargs: additional parameters to be passed to scoring functions 353 354 Returns: 355 356 model scores: {array-like} 357 358 """ 359 360 preds = self.predict(X) 361 362 if scoring is None: 363 scoring = "accuracy" 364 365 # check inputs 366 assert scoring in ( 367 "accuracy", 368 "average_precision", 369 "brier_score_loss", 370 "f1", 371 "f1_micro", 372 "f1_macro", 373 "f1_weighted", 374 "f1_samples", 375 "neg_log_loss", 376 "precision", 377 "recall", 378 "roc_auc", 379 ), "'scoring' should be in ('accuracy', 'average_precision', \ 380 'brier_score_loss', 'f1', 'f1_micro', \ 381 'f1_macro', 'f1_weighted', 'f1_samples', \ 382 'neg_log_loss', 'precision', 'recall', \ 383 'roc_auc')" 384 385 scoring_options = { 386 "accuracy": skm2.accuracy_score, 387 "average_precision": skm2.average_precision_score, 388 "brier_score_loss": skm2.brier_score_loss, 389 "f1": skm2.f1_score, 390 "f1_micro": skm2.f1_score, 391 "f1_macro": skm2.f1_score, 392 "f1_weighted": skm2.f1_score, 393 "f1_samples": skm2.f1_score, 394 "neg_log_loss": skm2.log_loss, 395 "precision": skm2.precision_score, 396 "recall": skm2.recall_score, 397 "roc_auc": skm2.roc_auc_score, 398 } 399 400 return scoring_options[scoring](y, preds, **kwargs)
Generalized 'linear' models using quasi-randomized networks (classification)
Parameters:
n_hidden_features: int
number of nodes in the hidden layer
lambda1: float
regularization parameter for GLM coefficients on original features
alpha1: float
controls compromize between l1 and l2 norm of GLM coefficients on original features
lambda2: float
regularization parameter for GLM coefficients on nonlinear features
alpha2: float
controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not
(False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original predictors are included (True) in model's
fitting or not (False)
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
optimizer: object
optimizer, from class nnetsauce.Optimizer
seed: int
reproducibility seed for nodes_sim=='uniform'
Attributes:
beta_: vector
regression coefficients
Examples:
See https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_classification.py
94 def __init__( 95 self, 96 n_hidden_features=5, 97 lambda1=0.01, 98 alpha1=0.5, 99 lambda2=0.01, 100 alpha2=0.5, 101 family="expit", 102 activation_name="relu", 103 a=0.01, 104 nodes_sim="sobol", 105 bias=True, 106 dropout=0, 107 direct_link=True, 108 n_clusters=2, 109 cluster_encode=True, 110 type_clust="kmeans", 111 type_scaling=("std", "std", "std"), 112 optimizer=Optimizer(), 113 seed=123, 114 ): 115 super().__init__( 116 n_hidden_features=n_hidden_features, 117 lambda1=lambda1, 118 alpha1=alpha1, 119 lambda2=lambda2, 120 alpha2=alpha2, 121 activation_name=activation_name, 122 a=a, 123 nodes_sim=nodes_sim, 124 bias=bias, 125 dropout=dropout, 126 direct_link=direct_link, 127 n_clusters=n_clusters, 128 cluster_encode=cluster_encode, 129 type_clust=type_clust, 130 type_scaling=type_scaling, 131 optimizer=optimizer, 132 seed=seed, 133 ) 134 135 self.family = family
137 def logit_loss(self, Y, row_index, XB): 138 self.n_classes = Y.shape[1] # len(np.unique(y)) 139 # Y = mo.one_hot_encode2(y, self.n_classes) 140 # Y = self.optimizer.one_hot_encode(y, self.n_classes) 141 142 # max_double = 709.0 # only if softmax 143 # XB[XB > max_double] = max_double 144 XB[XB > 709.0] = 709.0 145 146 if row_index is None: 147 return -np.mean(np.sum(Y * XB, axis=1) - logsumexp(XB)) 148 149 return -np.mean(np.sum(Y[row_index, :] * XB, axis=1) - logsumexp(XB))
151 def expit_erf_loss(self, Y, row_index, XB): 152 # self.n_classes = len(np.unique(y)) 153 # Y = mo.one_hot_encode2(y, self.n_classes) 154 # Y = self.optimizer.one_hot_encode(y, self.n_classes) 155 self.n_classes = Y.shape[1] 156 157 if row_index is None: 158 return -np.mean(np.sum(Y * XB, axis=1) - logsumexp(XB)) 159 160 return -np.mean(np.sum(Y[row_index, :] * XB, axis=1) - logsumexp(XB))
162 def loss_func( 163 self, 164 beta, 165 group_index, 166 X, 167 Y, 168 y, 169 row_index=None, 170 type_loss="logit", 171 **kwargs 172 ): 173 res = { 174 "logit": self.logit_loss, 175 "expit": self.expit_erf_loss, 176 "erf": self.expit_erf_loss, 177 } 178 179 if row_index is None: 180 row_index = range(len(y)) 181 XB = self.compute_XB( 182 X, 183 beta=np.reshape(beta, (X.shape[1], self.n_classes), order="F"), 184 ) 185 186 return res[type_loss](Y, row_index, XB) + self.compute_penalty( 187 group_index=group_index, beta=beta 188 ) 189 190 XB = self.compute_XB( 191 X, 192 beta=np.reshape(beta, (X.shape[1], self.n_classes), order="F"), 193 row_index=row_index, 194 ) 195 196 return res[type_loss](Y, row_index, XB) + self.compute_penalty( 197 group_index=group_index, beta=beta 198 )
200 def fit(self, X, y, **kwargs): 201 """Fit GLM model to training data (X, y). 202 203 Args: 204 205 X: {array-like}, shape = [n_samples, n_features] 206 Training vectors, where n_samples is the number 207 of samples and n_features is the number of features. 208 209 y: array-like, shape = [n_samples] 210 Target values. 211 212 **kwargs: additional parameters to be passed to 213 self.cook_training_set or self.obj.fit 214 215 Returns: 216 217 self: object 218 219 """ 220 221 assert mx.is_factor( 222 y 223 ), "y must contain only integers" # change is_factor and subsampling everywhere 224 225 self.beta_ = None 226 227 n, p = X.shape 228 229 self.group_index = n * X.shape[1] 230 231 self.n_classes = len(np.unique(y)) 232 233 output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 234 235 # Y = mo.one_hot_encode2(output_y, self.n_classes) 236 Y = self.optimizer.one_hot_encode(output_y, self.n_classes) 237 238 # initialization 239 beta_ = np.linalg.lstsq(scaled_Z, Y, rcond=None)[0] 240 241 # optimization 242 # fit(self, loss_func, response, x0, **kwargs): 243 # loss_func(self, beta, group_index, X, y, 244 # row_index=None, type_loss="gaussian", 245 # **kwargs) 246 self.optimizer.fit( 247 self.loss_func, 248 response=y, 249 x0=beta_.flatten(order="F"), 250 group_index=self.group_index, 251 X=scaled_Z, 252 Y=Y, 253 y=y, 254 type_loss=self.family, 255 ) 256 257 self.beta_ = self.optimizer.results[0] 258 259 return self
Fit GLM model to training data (X, y).
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
261 def predict(self, X, **kwargs): 262 """Predict test data X. 263 264 Args: 265 266 X: {array-like}, shape = [n_samples, n_features] 267 Training vectors, where n_samples is the number 268 of samples and n_features is the number of features. 269 270 **kwargs: additional parameters to be passed to 271 self.cook_test_set 272 273 Returns: 274 275 model predictions: {array-like} 276 277 """ 278 279 return np.argmax(self.predict_proba(X, **kwargs), axis=1)
Predict test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
281 def predict_proba(self, X, **kwargs): 282 """Predict probabilities for test data X. 283 284 Args: 285 286 X: {array-like}, shape = [n_samples, n_features] 287 Training vectors, where n_samples is the number 288 of samples and n_features is the number of features. 289 290 **kwargs: additional parameters to be passed to 291 self.cook_test_set 292 293 Returns: 294 295 probability estimates for test data: {array-like} 296 297 """ 298 if len(X.shape) == 1: 299 n_features = X.shape[0] 300 new_X = mo.rbind( 301 X.reshape(1, n_features), 302 np.ones(n_features).reshape(1, n_features), 303 ) 304 305 Z = self.cook_test_set(new_X, **kwargs) 306 307 else: 308 Z = self.cook_test_set(X, **kwargs) 309 310 ZB = mo.safe_sparse_dot( 311 Z, 312 self.beta_.reshape( 313 self.n_classes, 314 X.shape[1] + self.n_hidden_features + self.n_clusters, 315 ).T, 316 ) 317 318 if self.family == "logit": 319 exp_ZB = np.exp(ZB) 320 321 return exp_ZB / exp_ZB.sum(axis=1)[:, None] 322 323 if self.family == "expit": 324 exp_ZB = expit(ZB) 325 326 return exp_ZB / exp_ZB.sum(axis=1)[:, None] 327 328 if self.family == "erf": 329 exp_ZB = 0.5 * (1 + erf(ZB)) 330 331 return exp_ZB / exp_ZB.sum(axis=1)[:, None]
Predict probabilities for test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
probability estimates for test data: {array-like}
333 def score(self, X, y, scoring=None, **kwargs): 334 """ Score the model on test set features X and response y. 335 336 Args: 337 338 X: {array-like}, shape = [n_samples, n_features] 339 Training vectors, where n_samples is the number 340 of samples and n_features is the number of features 341 342 y: array-like, shape = [n_samples] 343 Target values 344 345 scoring: str 346 must be in ('accuracy', 'average_precision', \ 347 'brier_score_loss', 'f1', 'f1_micro', \ 348 'f1_macro', 'f1_weighted', 'f1_samples', \ 349 'neg_log_loss', 'precision', 'recall', \ 350 'roc_auc') 351 352 **kwargs: additional parameters to be passed to scoring functions 353 354 Returns: 355 356 model scores: {array-like} 357 358 """ 359 360 preds = self.predict(X) 361 362 if scoring is None: 363 scoring = "accuracy" 364 365 # check inputs 366 assert scoring in ( 367 "accuracy", 368 "average_precision", 369 "brier_score_loss", 370 "f1", 371 "f1_micro", 372 "f1_macro", 373 "f1_weighted", 374 "f1_samples", 375 "neg_log_loss", 376 "precision", 377 "recall", 378 "roc_auc", 379 ), "'scoring' should be in ('accuracy', 'average_precision', \ 380 'brier_score_loss', 'f1', 'f1_micro', \ 381 'f1_macro', 'f1_weighted', 'f1_samples', \ 382 'neg_log_loss', 'precision', 'recall', \ 383 'roc_auc')" 384 385 scoring_options = { 386 "accuracy": skm2.accuracy_score, 387 "average_precision": skm2.average_precision_score, 388 "brier_score_loss": skm2.brier_score_loss, 389 "f1": skm2.f1_score, 390 "f1_micro": skm2.f1_score, 391 "f1_macro": skm2.f1_score, 392 "f1_weighted": skm2.f1_score, 393 "f1_samples": skm2.f1_score, 394 "neg_log_loss": skm2.log_loss, 395 "precision": skm2.precision_score, 396 "recall": skm2.recall_score, 397 "roc_auc": skm2.roc_auc_score, 398 } 399 400 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('accuracy', 'average_precision', 'brier_score_loss', 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'roc_auc')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.glm.glm.GLM
- lambda1
- alpha1
- lambda2
- alpha2
- optimizer
- beta_
- compute_XB
- compute_XB2
- penalty
- compute_penalty
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
14class GLMRegressor(GLM, RegressorMixin): 15 """Generalized 'linear' models using quasi-randomized networks (regression) 16 17 Attributes: 18 19 n_hidden_features: int 20 number of nodes in the hidden layer 21 22 lambda1: float 23 regularization parameter for GLM coefficients on original features 24 25 alpha1: float 26 controls compromize between l1 and l2 norm of GLM coefficients on original features 27 28 lambda2: float 29 regularization parameter for GLM coefficients on nonlinear features 30 31 alpha2: float 32 controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features 33 34 family: str 35 "gaussian", "laplace" or "poisson" (for now) 36 37 activation_name: str 38 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 39 40 a: float 41 hyperparameter for 'prelu' or 'elu' activation function 42 43 nodes_sim: str 44 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 45 'uniform' 46 47 bias: boolean 48 indicates if the hidden layer contains a bias term (True) or not 49 (False) 50 51 dropout: float 52 regularization parameter; (random) percentage of nodes dropped out 53 of the training 54 55 direct_link: boolean 56 indicates if the original predictors are included (True) in model's 57 fitting or not (False) 58 59 n_clusters: int 60 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: 61 no clustering) 62 63 cluster_encode: bool 64 defines how the variable containing clusters is treated (default is one-hot) 65 if `False`, then labels are used, without one-hot encoding 66 67 type_clust: str 68 type of clustering method: currently k-means ('kmeans') or Gaussian 69 Mixture Model ('gmm') 70 71 type_scaling: a tuple of 3 strings 72 scaling methods for inputs, hidden layer, and clustering respectively 73 (and when relevant). 74 Currently available: standardization ('std') or MinMax scaling ('minmax') 75 76 optimizer: object 77 optimizer, from class nnetsauce.utils.Optimizer 78 79 seed: int 80 reproducibility seed for nodes_sim=='uniform' 81 82 Attributes: 83 84 beta_: vector 85 regression coefficients 86 87 Examples: 88 89 See [https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_regression.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_regression.py) 90 91 """ 92 93 # construct the object ----- 94 95 def __init__( 96 self, 97 n_hidden_features=5, 98 lambda1=0.01, 99 alpha1=0.5, 100 lambda2=0.01, 101 alpha2=0.5, 102 family="gaussian", 103 activation_name="relu", 104 a=0.01, 105 nodes_sim="sobol", 106 bias=True, 107 dropout=0, 108 direct_link=True, 109 n_clusters=2, 110 cluster_encode=True, 111 type_clust="kmeans", 112 type_scaling=("std", "std", "std"), 113 optimizer=Optimizer(), 114 seed=123, 115 ): 116 super().__init__( 117 n_hidden_features=n_hidden_features, 118 lambda1=lambda1, 119 alpha1=alpha1, 120 lambda2=lambda2, 121 alpha2=alpha2, 122 activation_name=activation_name, 123 a=a, 124 nodes_sim=nodes_sim, 125 bias=bias, 126 dropout=dropout, 127 direct_link=direct_link, 128 n_clusters=n_clusters, 129 cluster_encode=cluster_encode, 130 type_clust=type_clust, 131 type_scaling=type_scaling, 132 optimizer=optimizer, 133 seed=seed, 134 ) 135 136 self.family = family 137 138 def gaussian_loss(self, y, row_index, XB): 139 return 0.5 * np.mean(np.square(y[row_index] - XB)) 140 141 def laplace_loss(self, y, row_index, XB): 142 return 0.5 * np.mean(np.abs(y[row_index] - XB)) 143 144 def poisson_loss(self, y, row_index, XB): 145 return -np.mean(y[row_index] * XB - np.exp(XB)) 146 147 def loss_func( 148 self, 149 beta, 150 group_index, 151 X, 152 y, 153 row_index=None, 154 type_loss="gaussian", 155 **kwargs 156 ): 157 res = { 158 "gaussian": self.gaussian_loss, 159 "laplace": self.laplace_loss, 160 "poisson": self.poisson_loss, 161 } 162 163 if row_index is None: 164 row_index = range(len(y)) 165 XB = self.compute_XB(X, beta=beta) 166 167 return res[type_loss](y, row_index, XB) + self.compute_penalty( 168 group_index=group_index, beta=beta 169 ) 170 171 XB = self.compute_XB(X, beta=beta, row_index=row_index) 172 173 return res[type_loss](y, row_index, XB) + self.compute_penalty( 174 group_index=group_index, beta=beta 175 ) 176 177 def fit(self, X, y, **kwargs): 178 """Fit GLM model to training data (X, y). 179 180 Args: 181 182 X: {array-like}, shape = [n_samples, n_features] 183 Training vectors, where n_samples is the number 184 of samples and n_features is the number of features. 185 186 y: array-like, shape = [n_samples] 187 Target values. 188 189 **kwargs: additional parameters to be passed to 190 self.cook_training_set or self.obj.fit 191 192 Returns: 193 194 self: object 195 196 """ 197 198 self.beta_ = None 199 200 self.n_iter = 0 201 202 n, self.group_index = X.shape 203 204 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 205 206 n_Z = scaled_Z.shape[0] 207 208 # initialization 209 beta_ = np.linalg.lstsq(scaled_Z, centered_y, rcond=None)[0] 210 211 # optimization 212 # fit(self, loss_func, response, x0, **kwargs): 213 # loss_func(self, beta, group_index, X, y, 214 # row_index=None, type_loss="gaussian", 215 # **kwargs) 216 self.optimizer.fit( 217 self.loss_func, 218 response=centered_y, 219 x0=beta_, 220 group_index=self.group_index, 221 X=scaled_Z, 222 y=centered_y, 223 type_loss=self.family, 224 **kwargs 225 ) 226 227 self.beta_ = self.optimizer.results[0] 228 229 return self 230 231 def predict(self, X, **kwargs): 232 """Predict test data X. 233 234 Args: 235 236 X: {array-like}, shape = [n_samples, n_features] 237 Training vectors, where n_samples is the number 238 of samples and n_features is the number of features. 239 240 **kwargs: additional parameters to be passed to 241 self.cook_test_set 242 243 Returns: 244 245 model predictions: {array-like} 246 247 """ 248 249 if len(X.shape) == 1: 250 n_features = X.shape[0] 251 new_X = mo.rbind( 252 X.reshape(1, n_features), 253 np.ones(n_features).reshape(1, n_features), 254 ) 255 256 return ( 257 self.y_mean_ 258 + np.dot(self.cook_test_set(new_X, **kwargs), self.beta_) 259 )[0] 260 261 return self.y_mean_ + np.dot( 262 self.cook_test_set(X, **kwargs), self.beta_ 263 ) 264 265 def score(self, X, y, scoring=None, **kwargs): 266 """ Score the model on test set features X and response y. 267 268 Args: 269 270 X: {array-like}, shape = [n_samples, n_features] 271 Training vectors, where n_samples is the number 272 of samples and n_features is the number of features 273 274 y: array-like, shape = [n_samples] 275 Target values 276 277 scoring: str 278 must be in ('explained_variance', 'neg_mean_absolute_error', \ 279 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 280 'neg_median_absolute_error', 'r2') 281 282 **kwargs: additional parameters to be passed to scoring functions 283 284 Returns: 285 286 model scores: {array-like} 287 288 """ 289 290 preds = self.predict(X) 291 292 if type(preds) == tuple: # if there are std. devs in the predictions 293 preds = preds[0] 294 295 if scoring is None: 296 scoring = "neg_mean_squared_error" 297 298 # check inputs 299 assert scoring in ( 300 "explained_variance", 301 "neg_mean_absolute_error", 302 "neg_mean_squared_error", 303 "neg_mean_squared_log_error", 304 "neg_median_absolute_error", 305 "r2", 306 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 307 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 308 'neg_median_absolute_error', 'r2')" 309 310 scoring_options = { 311 "explained_variance": skm2.explained_variance_score, 312 "neg_mean_absolute_error": skm2.mean_absolute_error, 313 "neg_mean_squared_error": skm2.mean_squared_error, 314 "neg_mean_squared_log_error": skm2.mean_squared_log_error, 315 "neg_median_absolute_error": skm2.median_absolute_error, 316 "r2": skm2.r2_score, 317 } 318 319 return scoring_options[scoring](y, preds, **kwargs)
Generalized 'linear' models using quasi-randomized networks (regression)
Attributes:
n_hidden_features: int
number of nodes in the hidden layer
lambda1: float
regularization parameter for GLM coefficients on original features
alpha1: float
controls compromize between l1 and l2 norm of GLM coefficients on original features
lambda2: float
regularization parameter for GLM coefficients on nonlinear features
alpha2: float
controls compromize between l1 and l2 norm of GLM coefficients on nonlinear features
family: str
"gaussian", "laplace" or "poisson" (for now)
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not
(False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original predictors are included (True) in model's
fitting or not (False)
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
optimizer: object
optimizer, from class nnetsauce.utils.Optimizer
seed: int
reproducibility seed for nodes_sim=='uniform'
Attributes:
beta_: vector
regression coefficients
Examples:
See https://github.com/Techtonique/nnetsauce/blob/master/examples/glm_regression.py
95 def __init__( 96 self, 97 n_hidden_features=5, 98 lambda1=0.01, 99 alpha1=0.5, 100 lambda2=0.01, 101 alpha2=0.5, 102 family="gaussian", 103 activation_name="relu", 104 a=0.01, 105 nodes_sim="sobol", 106 bias=True, 107 dropout=0, 108 direct_link=True, 109 n_clusters=2, 110 cluster_encode=True, 111 type_clust="kmeans", 112 type_scaling=("std", "std", "std"), 113 optimizer=Optimizer(), 114 seed=123, 115 ): 116 super().__init__( 117 n_hidden_features=n_hidden_features, 118 lambda1=lambda1, 119 alpha1=alpha1, 120 lambda2=lambda2, 121 alpha2=alpha2, 122 activation_name=activation_name, 123 a=a, 124 nodes_sim=nodes_sim, 125 bias=bias, 126 dropout=dropout, 127 direct_link=direct_link, 128 n_clusters=n_clusters, 129 cluster_encode=cluster_encode, 130 type_clust=type_clust, 131 type_scaling=type_scaling, 132 optimizer=optimizer, 133 seed=seed, 134 ) 135 136 self.family = family
147 def loss_func( 148 self, 149 beta, 150 group_index, 151 X, 152 y, 153 row_index=None, 154 type_loss="gaussian", 155 **kwargs 156 ): 157 res = { 158 "gaussian": self.gaussian_loss, 159 "laplace": self.laplace_loss, 160 "poisson": self.poisson_loss, 161 } 162 163 if row_index is None: 164 row_index = range(len(y)) 165 XB = self.compute_XB(X, beta=beta) 166 167 return res[type_loss](y, row_index, XB) + self.compute_penalty( 168 group_index=group_index, beta=beta 169 ) 170 171 XB = self.compute_XB(X, beta=beta, row_index=row_index) 172 173 return res[type_loss](y, row_index, XB) + self.compute_penalty( 174 group_index=group_index, beta=beta 175 )
177 def fit(self, X, y, **kwargs): 178 """Fit GLM model to training data (X, y). 179 180 Args: 181 182 X: {array-like}, shape = [n_samples, n_features] 183 Training vectors, where n_samples is the number 184 of samples and n_features is the number of features. 185 186 y: array-like, shape = [n_samples] 187 Target values. 188 189 **kwargs: additional parameters to be passed to 190 self.cook_training_set or self.obj.fit 191 192 Returns: 193 194 self: object 195 196 """ 197 198 self.beta_ = None 199 200 self.n_iter = 0 201 202 n, self.group_index = X.shape 203 204 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 205 206 n_Z = scaled_Z.shape[0] 207 208 # initialization 209 beta_ = np.linalg.lstsq(scaled_Z, centered_y, rcond=None)[0] 210 211 # optimization 212 # fit(self, loss_func, response, x0, **kwargs): 213 # loss_func(self, beta, group_index, X, y, 214 # row_index=None, type_loss="gaussian", 215 # **kwargs) 216 self.optimizer.fit( 217 self.loss_func, 218 response=centered_y, 219 x0=beta_, 220 group_index=self.group_index, 221 X=scaled_Z, 222 y=centered_y, 223 type_loss=self.family, 224 **kwargs 225 ) 226 227 self.beta_ = self.optimizer.results[0] 228 229 return self
Fit GLM model to training data (X, y).
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
231 def predict(self, X, **kwargs): 232 """Predict test data X. 233 234 Args: 235 236 X: {array-like}, shape = [n_samples, n_features] 237 Training vectors, where n_samples is the number 238 of samples and n_features is the number of features. 239 240 **kwargs: additional parameters to be passed to 241 self.cook_test_set 242 243 Returns: 244 245 model predictions: {array-like} 246 247 """ 248 249 if len(X.shape) == 1: 250 n_features = X.shape[0] 251 new_X = mo.rbind( 252 X.reshape(1, n_features), 253 np.ones(n_features).reshape(1, n_features), 254 ) 255 256 return ( 257 self.y_mean_ 258 + np.dot(self.cook_test_set(new_X, **kwargs), self.beta_) 259 )[0] 260 261 return self.y_mean_ + np.dot( 262 self.cook_test_set(X, **kwargs), self.beta_ 263 )
Predict test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
265 def score(self, X, y, scoring=None, **kwargs): 266 """ Score the model on test set features X and response y. 267 268 Args: 269 270 X: {array-like}, shape = [n_samples, n_features] 271 Training vectors, where n_samples is the number 272 of samples and n_features is the number of features 273 274 y: array-like, shape = [n_samples] 275 Target values 276 277 scoring: str 278 must be in ('explained_variance', 'neg_mean_absolute_error', \ 279 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 280 'neg_median_absolute_error', 'r2') 281 282 **kwargs: additional parameters to be passed to scoring functions 283 284 Returns: 285 286 model scores: {array-like} 287 288 """ 289 290 preds = self.predict(X) 291 292 if type(preds) == tuple: # if there are std. devs in the predictions 293 preds = preds[0] 294 295 if scoring is None: 296 scoring = "neg_mean_squared_error" 297 298 # check inputs 299 assert scoring in ( 300 "explained_variance", 301 "neg_mean_absolute_error", 302 "neg_mean_squared_error", 303 "neg_mean_squared_log_error", 304 "neg_median_absolute_error", 305 "r2", 306 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 307 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 308 'neg_median_absolute_error', 'r2')" 309 310 scoring_options = { 311 "explained_variance": skm2.explained_variance_score, 312 "neg_mean_absolute_error": skm2.mean_absolute_error, 313 "neg_mean_squared_error": skm2.mean_squared_error, 314 "neg_mean_squared_log_error": skm2.mean_squared_log_error, 315 "neg_median_absolute_error": skm2.median_absolute_error, 316 "r2": skm2.r2_score, 317 } 318 319 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.glm.glm.GLM
- lambda1
- alpha1
- lambda2
- alpha2
- optimizer
- beta_
- compute_XB
- compute_XB2
- penalty
- compute_penalty
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
84class LazyClassifier(Custom, ClassifierMixin): 85 """ 86 This module helps in fitting to all the classification algorithms that are available in Scikit-learn to nnetsauce's CustomClassifier 87 Parameters 88 ---------- 89 verbose : int, optional (default=0) 90 For the liblinear and lbfgs solvers set verbose to any positive 91 number for verbosity. 92 ignore_warnings : bool, optional (default=True) 93 When set to True, the warning related to algorigms that are not able to run are ignored. 94 custom_metric : function, optional (default=None) 95 When function is provided, models are evaluated based on the custom evaluation metric provided. 96 predictions : bool, optional (default=False) 97 When set to True, the predictions of all the models models are returned as dataframe. 98 classifiers : list, optional (default="all") 99 When function is provided, trains the chosen classifier(s). 100 n_jobs : int, when possible, run in parallel 101 102 Examples 103 -------- 104 >>> import nnetsauce as ns 105 >>> from sklearn.datasets import load_breast_cancer 106 >>> from sklearn.model_selection import train_test_split 107 >>> data = load_breast_cancer() 108 >>> X = data.data 109 >>> y= data.target 110 >>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123) 111 >>> clf = ns.LazyClassifier(verbose=0, ignore_warnings=True, custom_metric=None) 112 >>> models, predictions = clf.fit(X_train, X_test, y_train, y_test) 113 >>> model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test) 114 >>> print(models) 115 """ 116 117 def __init__( 118 self, 119 verbose=0, 120 ignore_warnings=True, 121 custom_metric=None, 122 predictions=False, 123 random_state=42, 124 classifiers="all", 125 preprocess=False, 126 n_jobs=None, 127 # CustomClassifier attributes 128 obj=None, 129 n_hidden_features=5, 130 activation_name="relu", 131 a=0.01, 132 nodes_sim="sobol", 133 bias=True, 134 dropout=0, 135 direct_link=True, 136 n_clusters=2, 137 cluster_encode=True, 138 type_clust="kmeans", 139 type_scaling=("std", "std", "std"), 140 col_sample=1, 141 row_sample=1, 142 seed=123, 143 backend="cpu", 144 ): 145 self.verbose = verbose 146 self.ignore_warnings = ignore_warnings 147 self.custom_metric = custom_metric 148 self.predictions = predictions 149 self.models = {} 150 self.random_state = random_state 151 self.classifiers = classifiers 152 self.preprocess = preprocess 153 self.n_jobs = n_jobs 154 super().__init__( 155 obj=obj, 156 n_hidden_features=n_hidden_features, 157 activation_name=activation_name, 158 a=a, 159 nodes_sim=nodes_sim, 160 bias=bias, 161 dropout=dropout, 162 direct_link=direct_link, 163 n_clusters=n_clusters, 164 cluster_encode=cluster_encode, 165 type_clust=type_clust, 166 type_scaling=type_scaling, 167 col_sample=col_sample, 168 row_sample=row_sample, 169 seed=seed, 170 backend=backend, 171 ) 172 173 def fit(self, X_train, X_test, y_train, y_test): 174 """Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test. 175 Parameters 176 ---------- 177 X_train : array-like, 178 Training vectors, where rows is the number of samples 179 and columns is the number of features. 180 X_test : array-like, 181 Testing vectors, where rows is the number of samples 182 and columns is the number of features. 183 y_train : array-like, 184 Training vectors, where rows is the number of samples 185 and columns is the number of features. 186 y_test : array-like, 187 Testing vectors, where rows is the number of samples 188 and columns is the number of features. 189 Returns 190 ------- 191 scores : Pandas DataFrame 192 Returns metrics of all the models in a Pandas DataFrame. 193 predictions : Pandas DataFrame 194 Returns predictions of all the models in a Pandas DataFrame. 195 """ 196 Accuracy = [] 197 B_Accuracy = [] 198 ROC_AUC = [] 199 F1 = [] 200 names = [] 201 TIME = [] 202 predictions = {} 203 204 if self.custom_metric is not None: 205 CUSTOM_METRIC = [] 206 207 if isinstance(X_train, np.ndarray): 208 X_train = pd.DataFrame(X_train) 209 X_test = pd.DataFrame(X_test) 210 211 numeric_features = X_train.select_dtypes(include=[np.number]).columns 212 categorical_features = X_train.select_dtypes(include=["object"]).columns 213 214 categorical_low, categorical_high = get_card_split( 215 X_train, categorical_features 216 ) 217 218 if self.preprocess is True: 219 preprocessor = ColumnTransformer( 220 transformers=[ 221 ("numeric", numeric_transformer, numeric_features), 222 ( 223 "categorical_low", 224 categorical_transformer_low, 225 categorical_low, 226 ), 227 ( 228 "categorical_high", 229 categorical_transformer_high, 230 categorical_high, 231 ), 232 ] 233 ) 234 235 if self.classifiers == "all": 236 self.classifiers = [ 237 item 238 for sublist in [ 239 CLASSIFIERS, 240 MULTITASKCLASSIFIERS, 241 SIMPLEMULTITASKCLASSIFIERS, 242 ] 243 for item in sublist 244 ] 245 246 else: 247 try: 248 temp_list = [] 249 for classifier in self.classifiers: 250 full_name = (classifier.__name__, classifier) 251 temp_list.append(full_name) 252 self.classifiers = temp_list 253 except Exception as exception: 254 print(exception) 255 print("Invalid Classifier(s)") 256 257 if self.preprocess is True: 258 for name, model in tqdm(self.classifiers): # do parallel exec 259 other_args = ( 260 {} 261 ) # use this trick for `random_state` too --> refactor 262 try: 263 if ( 264 "n_jobs" in model().get_params().keys() 265 and name.find("LogisticRegression") == -1 266 ): 267 other_args["n_jobs"] = self.n_jobs 268 except Exception: 269 pass 270 271 start = time.time() 272 273 try: 274 if "random_state" in model().get_params().keys(): 275 pipe = Pipeline( 276 [ 277 ("preprocessor", preprocessor), 278 ( 279 "classifier", 280 CustomClassifier( 281 obj=model( 282 random_state=self.random_state, 283 **other_args 284 ), 285 n_hidden_features=self.n_hidden_features, 286 activation_name=self.activation_name, 287 a=self.a, 288 nodes_sim=self.nodes_sim, 289 bias=self.bias, 290 dropout=self.dropout, 291 direct_link=self.direct_link, 292 n_clusters=self.n_clusters, 293 cluster_encode=self.cluster_encode, 294 type_clust=self.type_clust, 295 type_scaling=self.type_scaling, 296 col_sample=self.col_sample, 297 row_sample=self.row_sample, 298 seed=self.seed, 299 backend=self.backend, 300 ), 301 ), 302 ] 303 ) 304 305 else: 306 pipe = Pipeline( 307 [ 308 ("preprocessor", preprocessor), 309 ( 310 "classifier", 311 CustomClassifier( 312 obj=model(**other_args), 313 n_hidden_features=self.n_hidden_features, 314 activation_name=self.activation_name, 315 a=self.a, 316 nodes_sim=self.nodes_sim, 317 bias=self.bias, 318 dropout=self.dropout, 319 direct_link=self.direct_link, 320 n_clusters=self.n_clusters, 321 cluster_encode=self.cluster_encode, 322 type_clust=self.type_clust, 323 type_scaling=self.type_scaling, 324 col_sample=self.col_sample, 325 row_sample=self.row_sample, 326 seed=self.seed, 327 backend=self.backend, 328 ), 329 ), 330 ] 331 ) 332 333 pipe.fit(X_train, y_train) 334 self.models[name] = pipe 335 y_pred = pipe.predict(X_test) 336 accuracy = accuracy_score(y_test, y_pred, normalize=True) 337 b_accuracy = balanced_accuracy_score(y_test, y_pred) 338 f1 = f1_score(y_test, y_pred, average="weighted") 339 try: 340 roc_auc = roc_auc_score(y_test, y_pred) 341 except Exception as exception: 342 roc_auc = None 343 if self.ignore_warnings is False: 344 print("ROC AUC couldn't be calculated for " + name) 345 print(exception) 346 names.append(name) 347 Accuracy.append(accuracy) 348 B_Accuracy.append(b_accuracy) 349 ROC_AUC.append(roc_auc) 350 F1.append(f1) 351 TIME.append(time.time() - start) 352 if self.custom_metric is not None: 353 custom_metric = self.custom_metric(y_test, y_pred) 354 CUSTOM_METRIC.append(custom_metric) 355 if self.verbose > 0: 356 if self.custom_metric is not None: 357 print( 358 { 359 "Model": name, 360 "Accuracy": accuracy, 361 "Balanced Accuracy": b_accuracy, 362 "ROC AUC": roc_auc, 363 "F1 Score": f1, 364 self.custom_metric.__name__: custom_metric, 365 "Time taken": time.time() - start, 366 } 367 ) 368 else: 369 print( 370 { 371 "Model": name, 372 "Accuracy": accuracy, 373 "Balanced Accuracy": b_accuracy, 374 "ROC AUC": roc_auc, 375 "F1 Score": f1, 376 "Time taken": time.time() - start, 377 } 378 ) 379 if self.predictions: 380 predictions[name] = y_pred 381 except Exception as exception: 382 if self.ignore_warnings is False: 383 print(name + " model failed to execute") 384 print(exception) 385 finally: 386 continue 387 388 else: 389 for name, model in tqdm(self.classifiers): # do parallel exec 390 other_args = ( 391 {} 392 ) # use this trick for `random_state` too --> refactor 393 try: 394 if ( 395 "n_jobs" in model().get_params().keys() 396 and name.find("LogisticRegression") == -1 397 ): 398 other_args["n_jobs"] = self.n_jobs 399 except Exception: 400 pass 401 402 start = time.time() 403 try: 404 if "random_state" in model().get_params().keys(): 405 pipe = CustomClassifier( 406 obj=model( 407 random_state=self.random_state, **other_args 408 ), 409 n_hidden_features=self.n_hidden_features, 410 activation_name=self.activation_name, 411 a=self.a, 412 nodes_sim=self.nodes_sim, 413 bias=self.bias, 414 dropout=self.dropout, 415 direct_link=self.direct_link, 416 n_clusters=self.n_clusters, 417 cluster_encode=self.cluster_encode, 418 type_clust=self.type_clust, 419 type_scaling=self.type_scaling, 420 col_sample=self.col_sample, 421 row_sample=self.row_sample, 422 seed=self.seed, 423 backend=self.backend, 424 ) 425 else: 426 pipe = CustomClassifier( 427 obj=model(**other_args), 428 n_hidden_features=self.n_hidden_features, 429 activation_name=self.activation_name, 430 a=self.a, 431 nodes_sim=self.nodes_sim, 432 bias=self.bias, 433 dropout=self.dropout, 434 direct_link=self.direct_link, 435 n_clusters=self.n_clusters, 436 cluster_encode=self.cluster_encode, 437 type_clust=self.type_clust, 438 type_scaling=self.type_scaling, 439 col_sample=self.col_sample, 440 row_sample=self.row_sample, 441 seed=self.seed, 442 backend=self.backend, 443 ) 444 445 pipe.fit(X_train, y_train) 446 self.models[name] = pipe 447 y_pred = pipe.predict(X_test) 448 accuracy = accuracy_score(y_test, y_pred, normalize=True) 449 b_accuracy = balanced_accuracy_score(y_test, y_pred) 450 f1 = f1_score(y_test, y_pred, average="weighted") 451 try: 452 roc_auc = roc_auc_score(y_test, y_pred) 453 except Exception as exception: 454 roc_auc = None 455 if self.ignore_warnings is False: 456 print("ROC AUC couldn't be calculated for " + name) 457 print(exception) 458 names.append(name) 459 Accuracy.append(accuracy) 460 B_Accuracy.append(b_accuracy) 461 ROC_AUC.append(roc_auc) 462 F1.append(f1) 463 TIME.append(time.time() - start) 464 if self.custom_metric is not None: 465 custom_metric = self.custom_metric(y_test, y_pred) 466 CUSTOM_METRIC.append(custom_metric) 467 if self.verbose > 0: 468 if self.custom_metric is not None: 469 print( 470 { 471 "Model": name, 472 "Accuracy": accuracy, 473 "Balanced Accuracy": b_accuracy, 474 "ROC AUC": roc_auc, 475 "F1 Score": f1, 476 self.custom_metric.__name__: custom_metric, 477 "Time taken": time.time() - start, 478 } 479 ) 480 else: 481 print( 482 { 483 "Model": name, 484 "Accuracy": accuracy, 485 "Balanced Accuracy": b_accuracy, 486 "ROC AUC": roc_auc, 487 "F1 Score": f1, 488 "Time taken": time.time() - start, 489 } 490 ) 491 if self.predictions: 492 predictions[name] = y_pred 493 except Exception as exception: 494 if self.ignore_warnings is False: 495 print(name + " model failed to execute") 496 print(exception) 497 finally: 498 continue 499 500 if self.custom_metric is None: 501 scores = pd.DataFrame( 502 { 503 "Model": names, 504 "Accuracy": Accuracy, 505 "Balanced Accuracy": B_Accuracy, 506 "ROC AUC": ROC_AUC, 507 "F1 Score": F1, 508 "Time Taken": TIME, 509 } 510 ) 511 else: 512 scores = pd.DataFrame( 513 { 514 "Model": names, 515 "Accuracy": Accuracy, 516 "Balanced Accuracy": B_Accuracy, 517 "ROC AUC": ROC_AUC, 518 "F1 Score": F1, 519 self.custom_metric.__name__: CUSTOM_METRIC, 520 "Time Taken": TIME, 521 } 522 ) 523 scores = scores.sort_values(by="Accuracy", ascending=False).set_index( 524 "Model" 525 ) 526 527 if self.predictions: 528 predictions_df = pd.DataFrame.from_dict(predictions) 529 return scores, predictions_df if self.predictions is True else scores 530 531 def provide_models(self, X_train, X_test, y_train, y_test): 532 """ 533 This function returns all the model objects trained in fit function. 534 If fit is not called already, then we call fit and then return the models. 535 Parameters 536 ---------- 537 X_train : array-like, 538 Training vectors, where rows is the number of samples 539 and columns is the number of features. 540 X_test : array-like, 541 Testing vectors, where rows is the number of samples 542 and columns is the number of features. 543 y_train : array-like, 544 Training vectors, where rows is the number of samples 545 and columns is the number of features. 546 y_test : array-like, 547 Testing vectors, where rows is the number of samples 548 and columns is the number of features. 549 Returns 550 ------- 551 models: dict-object, 552 Returns a dictionary with each model pipeline as value 553 with key as name of models. 554 """ 555 if len(self.models.keys()) == 0: 556 self.fit(X_train, X_test, y_train, y_test) 557 558 return self.models
This module helps in fitting to all the classification algorithms that are available in Scikit-learn to nnetsauce's CustomClassifier
Parameters
verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. predictions : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. classifiers : list, optional (default="all") When function is provided, trains the chosen classifier(s). n_jobs : int, when possible, run in parallel
Examples
>>> import nnetsauce as ns
>>> from sklearn.datasets import load_breast_cancer
>>> from sklearn.model_selection import train_test_split
>>> data = load_breast_cancer()
>>> X = data.data
>>> y= data.target
>>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
>>> clf = ns.LazyClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
>>> models, predictions = clf.fit(X_train, X_test, y_train, y_test)
>>> model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
>>> print(models)
117 def __init__( 118 self, 119 verbose=0, 120 ignore_warnings=True, 121 custom_metric=None, 122 predictions=False, 123 random_state=42, 124 classifiers="all", 125 preprocess=False, 126 n_jobs=None, 127 # CustomClassifier attributes 128 obj=None, 129 n_hidden_features=5, 130 activation_name="relu", 131 a=0.01, 132 nodes_sim="sobol", 133 bias=True, 134 dropout=0, 135 direct_link=True, 136 n_clusters=2, 137 cluster_encode=True, 138 type_clust="kmeans", 139 type_scaling=("std", "std", "std"), 140 col_sample=1, 141 row_sample=1, 142 seed=123, 143 backend="cpu", 144 ): 145 self.verbose = verbose 146 self.ignore_warnings = ignore_warnings 147 self.custom_metric = custom_metric 148 self.predictions = predictions 149 self.models = {} 150 self.random_state = random_state 151 self.classifiers = classifiers 152 self.preprocess = preprocess 153 self.n_jobs = n_jobs 154 super().__init__( 155 obj=obj, 156 n_hidden_features=n_hidden_features, 157 activation_name=activation_name, 158 a=a, 159 nodes_sim=nodes_sim, 160 bias=bias, 161 dropout=dropout, 162 direct_link=direct_link, 163 n_clusters=n_clusters, 164 cluster_encode=cluster_encode, 165 type_clust=type_clust, 166 type_scaling=type_scaling, 167 col_sample=col_sample, 168 row_sample=row_sample, 169 seed=seed, 170 backend=backend, 171 )
173 def fit(self, X_train, X_test, y_train, y_test): 174 """Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test. 175 Parameters 176 ---------- 177 X_train : array-like, 178 Training vectors, where rows is the number of samples 179 and columns is the number of features. 180 X_test : array-like, 181 Testing vectors, where rows is the number of samples 182 and columns is the number of features. 183 y_train : array-like, 184 Training vectors, where rows is the number of samples 185 and columns is the number of features. 186 y_test : array-like, 187 Testing vectors, where rows is the number of samples 188 and columns is the number of features. 189 Returns 190 ------- 191 scores : Pandas DataFrame 192 Returns metrics of all the models in a Pandas DataFrame. 193 predictions : Pandas DataFrame 194 Returns predictions of all the models in a Pandas DataFrame. 195 """ 196 Accuracy = [] 197 B_Accuracy = [] 198 ROC_AUC = [] 199 F1 = [] 200 names = [] 201 TIME = [] 202 predictions = {} 203 204 if self.custom_metric is not None: 205 CUSTOM_METRIC = [] 206 207 if isinstance(X_train, np.ndarray): 208 X_train = pd.DataFrame(X_train) 209 X_test = pd.DataFrame(X_test) 210 211 numeric_features = X_train.select_dtypes(include=[np.number]).columns 212 categorical_features = X_train.select_dtypes(include=["object"]).columns 213 214 categorical_low, categorical_high = get_card_split( 215 X_train, categorical_features 216 ) 217 218 if self.preprocess is True: 219 preprocessor = ColumnTransformer( 220 transformers=[ 221 ("numeric", numeric_transformer, numeric_features), 222 ( 223 "categorical_low", 224 categorical_transformer_low, 225 categorical_low, 226 ), 227 ( 228 "categorical_high", 229 categorical_transformer_high, 230 categorical_high, 231 ), 232 ] 233 ) 234 235 if self.classifiers == "all": 236 self.classifiers = [ 237 item 238 for sublist in [ 239 CLASSIFIERS, 240 MULTITASKCLASSIFIERS, 241 SIMPLEMULTITASKCLASSIFIERS, 242 ] 243 for item in sublist 244 ] 245 246 else: 247 try: 248 temp_list = [] 249 for classifier in self.classifiers: 250 full_name = (classifier.__name__, classifier) 251 temp_list.append(full_name) 252 self.classifiers = temp_list 253 except Exception as exception: 254 print(exception) 255 print("Invalid Classifier(s)") 256 257 if self.preprocess is True: 258 for name, model in tqdm(self.classifiers): # do parallel exec 259 other_args = ( 260 {} 261 ) # use this trick for `random_state` too --> refactor 262 try: 263 if ( 264 "n_jobs" in model().get_params().keys() 265 and name.find("LogisticRegression") == -1 266 ): 267 other_args["n_jobs"] = self.n_jobs 268 except Exception: 269 pass 270 271 start = time.time() 272 273 try: 274 if "random_state" in model().get_params().keys(): 275 pipe = Pipeline( 276 [ 277 ("preprocessor", preprocessor), 278 ( 279 "classifier", 280 CustomClassifier( 281 obj=model( 282 random_state=self.random_state, 283 **other_args 284 ), 285 n_hidden_features=self.n_hidden_features, 286 activation_name=self.activation_name, 287 a=self.a, 288 nodes_sim=self.nodes_sim, 289 bias=self.bias, 290 dropout=self.dropout, 291 direct_link=self.direct_link, 292 n_clusters=self.n_clusters, 293 cluster_encode=self.cluster_encode, 294 type_clust=self.type_clust, 295 type_scaling=self.type_scaling, 296 col_sample=self.col_sample, 297 row_sample=self.row_sample, 298 seed=self.seed, 299 backend=self.backend, 300 ), 301 ), 302 ] 303 ) 304 305 else: 306 pipe = Pipeline( 307 [ 308 ("preprocessor", preprocessor), 309 ( 310 "classifier", 311 CustomClassifier( 312 obj=model(**other_args), 313 n_hidden_features=self.n_hidden_features, 314 activation_name=self.activation_name, 315 a=self.a, 316 nodes_sim=self.nodes_sim, 317 bias=self.bias, 318 dropout=self.dropout, 319 direct_link=self.direct_link, 320 n_clusters=self.n_clusters, 321 cluster_encode=self.cluster_encode, 322 type_clust=self.type_clust, 323 type_scaling=self.type_scaling, 324 col_sample=self.col_sample, 325 row_sample=self.row_sample, 326 seed=self.seed, 327 backend=self.backend, 328 ), 329 ), 330 ] 331 ) 332 333 pipe.fit(X_train, y_train) 334 self.models[name] = pipe 335 y_pred = pipe.predict(X_test) 336 accuracy = accuracy_score(y_test, y_pred, normalize=True) 337 b_accuracy = balanced_accuracy_score(y_test, y_pred) 338 f1 = f1_score(y_test, y_pred, average="weighted") 339 try: 340 roc_auc = roc_auc_score(y_test, y_pred) 341 except Exception as exception: 342 roc_auc = None 343 if self.ignore_warnings is False: 344 print("ROC AUC couldn't be calculated for " + name) 345 print(exception) 346 names.append(name) 347 Accuracy.append(accuracy) 348 B_Accuracy.append(b_accuracy) 349 ROC_AUC.append(roc_auc) 350 F1.append(f1) 351 TIME.append(time.time() - start) 352 if self.custom_metric is not None: 353 custom_metric = self.custom_metric(y_test, y_pred) 354 CUSTOM_METRIC.append(custom_metric) 355 if self.verbose > 0: 356 if self.custom_metric is not None: 357 print( 358 { 359 "Model": name, 360 "Accuracy": accuracy, 361 "Balanced Accuracy": b_accuracy, 362 "ROC AUC": roc_auc, 363 "F1 Score": f1, 364 self.custom_metric.__name__: custom_metric, 365 "Time taken": time.time() - start, 366 } 367 ) 368 else: 369 print( 370 { 371 "Model": name, 372 "Accuracy": accuracy, 373 "Balanced Accuracy": b_accuracy, 374 "ROC AUC": roc_auc, 375 "F1 Score": f1, 376 "Time taken": time.time() - start, 377 } 378 ) 379 if self.predictions: 380 predictions[name] = y_pred 381 except Exception as exception: 382 if self.ignore_warnings is False: 383 print(name + " model failed to execute") 384 print(exception) 385 finally: 386 continue 387 388 else: 389 for name, model in tqdm(self.classifiers): # do parallel exec 390 other_args = ( 391 {} 392 ) # use this trick for `random_state` too --> refactor 393 try: 394 if ( 395 "n_jobs" in model().get_params().keys() 396 and name.find("LogisticRegression") == -1 397 ): 398 other_args["n_jobs"] = self.n_jobs 399 except Exception: 400 pass 401 402 start = time.time() 403 try: 404 if "random_state" in model().get_params().keys(): 405 pipe = CustomClassifier( 406 obj=model( 407 random_state=self.random_state, **other_args 408 ), 409 n_hidden_features=self.n_hidden_features, 410 activation_name=self.activation_name, 411 a=self.a, 412 nodes_sim=self.nodes_sim, 413 bias=self.bias, 414 dropout=self.dropout, 415 direct_link=self.direct_link, 416 n_clusters=self.n_clusters, 417 cluster_encode=self.cluster_encode, 418 type_clust=self.type_clust, 419 type_scaling=self.type_scaling, 420 col_sample=self.col_sample, 421 row_sample=self.row_sample, 422 seed=self.seed, 423 backend=self.backend, 424 ) 425 else: 426 pipe = CustomClassifier( 427 obj=model(**other_args), 428 n_hidden_features=self.n_hidden_features, 429 activation_name=self.activation_name, 430 a=self.a, 431 nodes_sim=self.nodes_sim, 432 bias=self.bias, 433 dropout=self.dropout, 434 direct_link=self.direct_link, 435 n_clusters=self.n_clusters, 436 cluster_encode=self.cluster_encode, 437 type_clust=self.type_clust, 438 type_scaling=self.type_scaling, 439 col_sample=self.col_sample, 440 row_sample=self.row_sample, 441 seed=self.seed, 442 backend=self.backend, 443 ) 444 445 pipe.fit(X_train, y_train) 446 self.models[name] = pipe 447 y_pred = pipe.predict(X_test) 448 accuracy = accuracy_score(y_test, y_pred, normalize=True) 449 b_accuracy = balanced_accuracy_score(y_test, y_pred) 450 f1 = f1_score(y_test, y_pred, average="weighted") 451 try: 452 roc_auc = roc_auc_score(y_test, y_pred) 453 except Exception as exception: 454 roc_auc = None 455 if self.ignore_warnings is False: 456 print("ROC AUC couldn't be calculated for " + name) 457 print(exception) 458 names.append(name) 459 Accuracy.append(accuracy) 460 B_Accuracy.append(b_accuracy) 461 ROC_AUC.append(roc_auc) 462 F1.append(f1) 463 TIME.append(time.time() - start) 464 if self.custom_metric is not None: 465 custom_metric = self.custom_metric(y_test, y_pred) 466 CUSTOM_METRIC.append(custom_metric) 467 if self.verbose > 0: 468 if self.custom_metric is not None: 469 print( 470 { 471 "Model": name, 472 "Accuracy": accuracy, 473 "Balanced Accuracy": b_accuracy, 474 "ROC AUC": roc_auc, 475 "F1 Score": f1, 476 self.custom_metric.__name__: custom_metric, 477 "Time taken": time.time() - start, 478 } 479 ) 480 else: 481 print( 482 { 483 "Model": name, 484 "Accuracy": accuracy, 485 "Balanced Accuracy": b_accuracy, 486 "ROC AUC": roc_auc, 487 "F1 Score": f1, 488 "Time taken": time.time() - start, 489 } 490 ) 491 if self.predictions: 492 predictions[name] = y_pred 493 except Exception as exception: 494 if self.ignore_warnings is False: 495 print(name + " model failed to execute") 496 print(exception) 497 finally: 498 continue 499 500 if self.custom_metric is None: 501 scores = pd.DataFrame( 502 { 503 "Model": names, 504 "Accuracy": Accuracy, 505 "Balanced Accuracy": B_Accuracy, 506 "ROC AUC": ROC_AUC, 507 "F1 Score": F1, 508 "Time Taken": TIME, 509 } 510 ) 511 else: 512 scores = pd.DataFrame( 513 { 514 "Model": names, 515 "Accuracy": Accuracy, 516 "Balanced Accuracy": B_Accuracy, 517 "ROC AUC": ROC_AUC, 518 "F1 Score": F1, 519 self.custom_metric.__name__: CUSTOM_METRIC, 520 "Time Taken": TIME, 521 } 522 ) 523 scores = scores.sort_values(by="Accuracy", ascending=False).set_index( 524 "Model" 525 ) 526 527 if self.predictions: 528 predictions_df = pd.DataFrame.from_dict(predictions) 529 return scores, predictions_df if self.predictions is True else scores
Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.
531 def provide_models(self, X_train, X_test, y_train, y_test): 532 """ 533 This function returns all the model objects trained in fit function. 534 If fit is not called already, then we call fit and then return the models. 535 Parameters 536 ---------- 537 X_train : array-like, 538 Training vectors, where rows is the number of samples 539 and columns is the number of features. 540 X_test : array-like, 541 Testing vectors, where rows is the number of samples 542 and columns is the number of features. 543 y_train : array-like, 544 Training vectors, where rows is the number of samples 545 and columns is the number of features. 546 y_test : array-like, 547 Testing vectors, where rows is the number of samples 548 and columns is the number of features. 549 Returns 550 ------- 551 models: dict-object, 552 Returns a dictionary with each model pipeline as value 553 with key as name of models. 554 """ 555 if len(self.models.keys()) == 0: 556 self.fit(X_train, X_test, y_train, y_test) 557 558 return self.models
This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.custom.custom.Custom
- obj
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
- sklearn.base.ClassifierMixin
- score
84class LazyRegressor(Custom, RegressorMixin): 85 """ 86 This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's CustomRegressor 87 Parameters 88 ---------- 89 verbose : int, optional (default=0) 90 For the liblinear and lbfgs solvers set verbose to any positive 91 number for verbosity. 92 ignore_warnings : bool, optional (default=True) 93 When set to True, the warning related to algorigms that are not able to run are ignored. 94 custom_metric : function, optional (default=None) 95 When function is provided, models are evaluated based on the custom evaluation metric provided. 96 prediction : bool, optional (default=False) 97 When set to True, the predictions of all the models models are returned as dataframe. 98 regressors : list, optional (default="all") 99 When function is provided, trains the chosen regressor(s). 100 n_jobs : int, when possible, run in parallel 101 102 Examples 103 -------- 104 >>> from lazypredict.Supervised import LazyRegressor 105 >>> from sklearn import datasets 106 >>> from sklearn.utils import shuffle 107 >>> import numpy as np 108 109 >>> diabetes = datasets.load_diabetes() 110 >>> X, y = shuffle(diabetes.data, diabetes.target, random_state=13) 111 >>> X = X.astype(np.float32) 112 113 >>> offset = int(X.shape[0] * 0.9) 114 >>> X_train, y_train = X[:offset], y[:offset] 115 >>> X_test, y_test = X[offset:], y[offset:] 116 117 >>> reg = LazyRegressor(verbose=0, ignore_warnings=False, custom_metric=None) 118 >>> models, predictions = reg.fit(X_train, X_test, y_train, y_test) 119 >>> model_dictionary = reg.provide_models(X_train, X_test, y_train, y_test) 120 >>> print(models) 121 """ 122 123 def __init__( 124 self, 125 verbose=0, 126 ignore_warnings=True, 127 custom_metric=None, 128 predictions=False, 129 random_state=42, 130 regressors="all", 131 preprocess=False, 132 n_jobs=None, 133 # CustomRegressor attributes 134 obj=None, 135 n_hidden_features=5, 136 activation_name="relu", 137 a=0.01, 138 nodes_sim="sobol", 139 bias=True, 140 dropout=0, 141 direct_link=True, 142 n_clusters=2, 143 cluster_encode=True, 144 type_clust="kmeans", 145 type_scaling=("std", "std", "std"), 146 col_sample=1, 147 row_sample=1, 148 seed=123, 149 backend="cpu", 150 ): 151 self.verbose = verbose 152 self.ignore_warnings = ignore_warnings 153 self.custom_metric = custom_metric 154 self.predictions = predictions 155 self.models = {} 156 self.random_state = random_state 157 self.regressors = regressors 158 self.preprocess = preprocess 159 self.n_jobs = n_jobs 160 super().__init__( 161 obj=obj, 162 n_hidden_features=n_hidden_features, 163 activation_name=activation_name, 164 a=a, 165 nodes_sim=nodes_sim, 166 bias=bias, 167 dropout=dropout, 168 direct_link=direct_link, 169 n_clusters=n_clusters, 170 cluster_encode=cluster_encode, 171 type_clust=type_clust, 172 type_scaling=type_scaling, 173 col_sample=col_sample, 174 row_sample=row_sample, 175 seed=seed, 176 backend=backend, 177 ) 178 179 def fit(self, X_train, X_test, y_train, y_test): 180 """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test. 181 Parameters 182 ---------- 183 X_train : array-like, 184 Training vectors, where rows is the number of samples 185 and columns is the number of features. 186 X_test : array-like, 187 Testing vectors, where rows is the number of samples 188 and columns is the number of features. 189 y_train : array-like, 190 Training vectors, where rows is the number of samples 191 and columns is the number of features. 192 y_test : array-like, 193 Testing vectors, where rows is the number of samples 194 and columns is the number of features. 195 Returns 196 ------- 197 scores : Pandas DataFrame 198 Returns metrics of all the models in a Pandas DataFrame. 199 predictions : Pandas DataFrame 200 Returns predictions of all the models in a Pandas DataFrame. 201 """ 202 R2 = [] 203 ADJR2 = [] 204 RMSE = [] 205 # WIN = [] 206 names = [] 207 TIME = [] 208 predictions = {} 209 210 if self.custom_metric: 211 CUSTOM_METRIC = [] 212 213 if isinstance(X_train, np.ndarray): 214 X_train = pd.DataFrame(X_train) 215 X_test = pd.DataFrame(X_test) 216 217 numeric_features = X_train.select_dtypes(include=[np.number]).columns 218 categorical_features = X_train.select_dtypes(include=["object"]).columns 219 220 categorical_low, categorical_high = get_card_split( 221 X_train, categorical_features 222 ) 223 224 if self.preprocess is True: 225 preprocessor = ColumnTransformer( 226 transformers=[ 227 ("numeric", numeric_transformer, numeric_features), 228 ( 229 "categorical_low", 230 categorical_transformer_low, 231 categorical_low, 232 ), 233 ( 234 "categorical_high", 235 categorical_transformer_high, 236 categorical_high, 237 ), 238 ] 239 ) 240 241 if self.regressors == "all": 242 self.regressors = REGRESSORS 243 else: 244 try: 245 temp_list = [] 246 for regressor in self.regressors: 247 full_name = (regressor.__name__, regressor) 248 temp_list.append(full_name) 249 self.regressors = temp_list 250 except Exception as exception: 251 print(exception) 252 print("Invalid Regressor(s)") 253 254 if self.preprocess is True: 255 for name, model in tqdm(self.regressors): # do parallel exec 256 start = time.time() 257 try: 258 if "random_state" in model().get_params().keys(): 259 pipe = Pipeline( 260 steps=[ 261 ("preprocessor", preprocessor), 262 ( 263 "regressor", 264 CustomRegressor( 265 obj=model( 266 random_state=self.random_state 267 ), 268 n_hidden_features=self.n_hidden_features, 269 activation_name=self.activation_name, 270 a=self.a, 271 nodes_sim=self.nodes_sim, 272 bias=self.bias, 273 dropout=self.dropout, 274 direct_link=self.direct_link, 275 n_clusters=self.n_clusters, 276 cluster_encode=self.cluster_encode, 277 type_clust=self.type_clust, 278 type_scaling=self.type_scaling, 279 col_sample=self.col_sample, 280 row_sample=self.row_sample, 281 seed=self.seed, 282 backend=self.backend, 283 ), 284 ), 285 ] 286 ) 287 else: 288 pipe = Pipeline( 289 steps=[ 290 ("preprocessor", preprocessor), 291 ( 292 "regressor", 293 CustomRegressor( 294 obj=model(), 295 n_hidden_features=self.n_hidden_features, 296 activation_name=self.activation_name, 297 a=self.a, 298 nodes_sim=self.nodes_sim, 299 bias=self.bias, 300 dropout=self.dropout, 301 direct_link=self.direct_link, 302 n_clusters=self.n_clusters, 303 cluster_encode=self.cluster_encode, 304 type_clust=self.type_clust, 305 type_scaling=self.type_scaling, 306 col_sample=self.col_sample, 307 row_sample=self.row_sample, 308 seed=self.seed, 309 backend=self.backend, 310 ), 311 ), 312 ] 313 ) 314 315 pipe.fit(X_train, y_train) 316 self.models[name] = pipe 317 y_pred = pipe.predict(X_test) 318 319 r_squared = r2_score(y_test, y_pred) 320 adj_rsquared = adjusted_rsquared( 321 r_squared, X_test.shape[0], X_test.shape[1] 322 ) 323 rmse = mean_squared_error(y_test, y_pred, squared=False) 324 325 names.append(name) 326 R2.append(r_squared) 327 ADJR2.append(adj_rsquared) 328 RMSE.append(rmse) 329 TIME.append(time.time() - start) 330 331 if self.custom_metric: 332 custom_metric = self.custom_metric(y_test, y_pred) 333 CUSTOM_METRIC.append(custom_metric) 334 335 if self.verbose > 0: 336 scores_verbose = { 337 "Model": name, 338 "R-Squared": r_squared, 339 "Adjusted R-Squared": adj_rsquared, 340 "RMSE": rmse, 341 "Time taken": time.time() - start, 342 } 343 344 if self.custom_metric: 345 scores_verbose[self.custom_metric.__name__] = ( 346 custom_metric 347 ) 348 349 print(scores_verbose) 350 if self.predictions: 351 predictions[name] = y_pred 352 except Exception as exception: 353 if self.ignore_warnings is False: 354 print(name + " model failed to execute") 355 print(exception) 356 357 else: 358 for name, model in tqdm(self.regressors): # do parallel exec 359 start = time.time() 360 try: 361 if "random_state" in model().get_params().keys(): 362 pipe = CustomRegressor( 363 obj=model(random_state=self.random_state), 364 n_hidden_features=self.n_hidden_features, 365 activation_name=self.activation_name, 366 a=self.a, 367 nodes_sim=self.nodes_sim, 368 bias=self.bias, 369 dropout=self.dropout, 370 direct_link=self.direct_link, 371 n_clusters=self.n_clusters, 372 cluster_encode=self.cluster_encode, 373 type_clust=self.type_clust, 374 type_scaling=self.type_scaling, 375 col_sample=self.col_sample, 376 row_sample=self.row_sample, 377 seed=self.seed, 378 backend=self.backend, 379 ) 380 else: 381 pipe = CustomRegressor( 382 obj=model(), 383 n_hidden_features=self.n_hidden_features, 384 activation_name=self.activation_name, 385 a=self.a, 386 nodes_sim=self.nodes_sim, 387 bias=self.bias, 388 dropout=self.dropout, 389 direct_link=self.direct_link, 390 n_clusters=self.n_clusters, 391 cluster_encode=self.cluster_encode, 392 type_clust=self.type_clust, 393 type_scaling=self.type_scaling, 394 col_sample=self.col_sample, 395 row_sample=self.row_sample, 396 seed=self.seed, 397 backend=self.backend, 398 ) 399 400 pipe.fit(X_train, y_train) 401 self.models[name] = pipe 402 y_pred = pipe.predict(X_test) 403 404 r_squared = r2_score(y_test, y_pred) 405 adj_rsquared = adjusted_rsquared( 406 r_squared, X_test.shape[0], X_test.shape[1] 407 ) 408 rmse = mean_squared_error(y_test, y_pred, squared=False) 409 410 names.append(name) 411 R2.append(r_squared) 412 ADJR2.append(adj_rsquared) 413 RMSE.append(rmse) 414 TIME.append(time.time() - start) 415 416 if self.custom_metric: 417 custom_metric = self.custom_metric(y_test, y_pred) 418 CUSTOM_METRIC.append(custom_metric) 419 420 if self.verbose > 0: 421 scores_verbose = { 422 "Model": name, 423 "R-Squared": r_squared, 424 "Adjusted R-Squared": adj_rsquared, 425 "RMSE": rmse, 426 "Time taken": time.time() - start, 427 } 428 429 if self.custom_metric: 430 scores_verbose[self.custom_metric.__name__] = ( 431 custom_metric 432 ) 433 434 print(scores_verbose) 435 if self.predictions: 436 predictions[name] = y_pred 437 except Exception as exception: 438 if self.ignore_warnings is False: 439 print(name + " model failed to execute") 440 print(exception) 441 442 scores = { 443 "Model": names, 444 "Adjusted R-Squared": ADJR2, 445 "R-Squared": R2, 446 "RMSE": RMSE, 447 "Time Taken": TIME, 448 } 449 450 if self.custom_metric: 451 scores[self.custom_metric.__name__] = CUSTOM_METRIC 452 453 scores = pd.DataFrame(scores) 454 scores = scores.sort_values(by="RMSE", ascending=True).set_index( 455 "Model" 456 ) 457 458 if self.predictions: 459 predictions_df = pd.DataFrame.from_dict(predictions) 460 return scores, predictions_df if self.predictions is True else scores 461 462 def provide_models(self, X_train, X_test, y_train, y_test): 463 """ 464 This function returns all the model objects trained in fit function. 465 If fit is not called already, then we call fit and then return the models. 466 Parameters 467 ---------- 468 X_train : array-like, 469 Training vectors, where rows is the number of samples 470 and columns is the number of features. 471 X_test : array-like, 472 Testing vectors, where rows is the number of samples 473 and columns is the number of features. 474 y_train : array-like, 475 Training vectors, where rows is the number of samples 476 and columns is the number of features. 477 y_test : array-like, 478 Testing vectors, where rows is the number of samples 479 and columns is the number of features. 480 Returns 481 ------- 482 models: dict-object, 483 Returns a dictionary with each model pipeline as value 484 with key as name of models. 485 """ 486 if len(self.models.keys()) == 0: 487 self.fit(X_train, X_test, y_train, y_test) 488 489 return self.models
This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's CustomRegressor
Parameters
verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. prediction : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. regressors : list, optional (default="all") When function is provided, trains the chosen regressor(s). n_jobs : int, when possible, run in parallel
Examples
>>> from lazypredict.Supervised import LazyRegressor
>>> from sklearn import datasets
>>> from sklearn.utils import shuffle
>>> import numpy as np
>>> diabetes = datasets.load_diabetes()
>>> X, y = shuffle(diabetes.data, diabetes.target, random_state=13)
>>> X = X.astype(np.float32)
>>> offset = int(X.shape[0] * 0.9)
>>> X_train, y_train = X[:offset], y[:offset]
>>> X_test, y_test = X[offset:], y[offset:]
>>> reg = LazyRegressor(verbose=0, ignore_warnings=False, custom_metric=None)
>>> models, predictions = reg.fit(X_train, X_test, y_train, y_test)
>>> model_dictionary = reg.provide_models(X_train, X_test, y_train, y_test)
>>> print(models)
123 def __init__( 124 self, 125 verbose=0, 126 ignore_warnings=True, 127 custom_metric=None, 128 predictions=False, 129 random_state=42, 130 regressors="all", 131 preprocess=False, 132 n_jobs=None, 133 # CustomRegressor attributes 134 obj=None, 135 n_hidden_features=5, 136 activation_name="relu", 137 a=0.01, 138 nodes_sim="sobol", 139 bias=True, 140 dropout=0, 141 direct_link=True, 142 n_clusters=2, 143 cluster_encode=True, 144 type_clust="kmeans", 145 type_scaling=("std", "std", "std"), 146 col_sample=1, 147 row_sample=1, 148 seed=123, 149 backend="cpu", 150 ): 151 self.verbose = verbose 152 self.ignore_warnings = ignore_warnings 153 self.custom_metric = custom_metric 154 self.predictions = predictions 155 self.models = {} 156 self.random_state = random_state 157 self.regressors = regressors 158 self.preprocess = preprocess 159 self.n_jobs = n_jobs 160 super().__init__( 161 obj=obj, 162 n_hidden_features=n_hidden_features, 163 activation_name=activation_name, 164 a=a, 165 nodes_sim=nodes_sim, 166 bias=bias, 167 dropout=dropout, 168 direct_link=direct_link, 169 n_clusters=n_clusters, 170 cluster_encode=cluster_encode, 171 type_clust=type_clust, 172 type_scaling=type_scaling, 173 col_sample=col_sample, 174 row_sample=row_sample, 175 seed=seed, 176 backend=backend, 177 )
179 def fit(self, X_train, X_test, y_train, y_test): 180 """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test. 181 Parameters 182 ---------- 183 X_train : array-like, 184 Training vectors, where rows is the number of samples 185 and columns is the number of features. 186 X_test : array-like, 187 Testing vectors, where rows is the number of samples 188 and columns is the number of features. 189 y_train : array-like, 190 Training vectors, where rows is the number of samples 191 and columns is the number of features. 192 y_test : array-like, 193 Testing vectors, where rows is the number of samples 194 and columns is the number of features. 195 Returns 196 ------- 197 scores : Pandas DataFrame 198 Returns metrics of all the models in a Pandas DataFrame. 199 predictions : Pandas DataFrame 200 Returns predictions of all the models in a Pandas DataFrame. 201 """ 202 R2 = [] 203 ADJR2 = [] 204 RMSE = [] 205 # WIN = [] 206 names = [] 207 TIME = [] 208 predictions = {} 209 210 if self.custom_metric: 211 CUSTOM_METRIC = [] 212 213 if isinstance(X_train, np.ndarray): 214 X_train = pd.DataFrame(X_train) 215 X_test = pd.DataFrame(X_test) 216 217 numeric_features = X_train.select_dtypes(include=[np.number]).columns 218 categorical_features = X_train.select_dtypes(include=["object"]).columns 219 220 categorical_low, categorical_high = get_card_split( 221 X_train, categorical_features 222 ) 223 224 if self.preprocess is True: 225 preprocessor = ColumnTransformer( 226 transformers=[ 227 ("numeric", numeric_transformer, numeric_features), 228 ( 229 "categorical_low", 230 categorical_transformer_low, 231 categorical_low, 232 ), 233 ( 234 "categorical_high", 235 categorical_transformer_high, 236 categorical_high, 237 ), 238 ] 239 ) 240 241 if self.regressors == "all": 242 self.regressors = REGRESSORS 243 else: 244 try: 245 temp_list = [] 246 for regressor in self.regressors: 247 full_name = (regressor.__name__, regressor) 248 temp_list.append(full_name) 249 self.regressors = temp_list 250 except Exception as exception: 251 print(exception) 252 print("Invalid Regressor(s)") 253 254 if self.preprocess is True: 255 for name, model in tqdm(self.regressors): # do parallel exec 256 start = time.time() 257 try: 258 if "random_state" in model().get_params().keys(): 259 pipe = Pipeline( 260 steps=[ 261 ("preprocessor", preprocessor), 262 ( 263 "regressor", 264 CustomRegressor( 265 obj=model( 266 random_state=self.random_state 267 ), 268 n_hidden_features=self.n_hidden_features, 269 activation_name=self.activation_name, 270 a=self.a, 271 nodes_sim=self.nodes_sim, 272 bias=self.bias, 273 dropout=self.dropout, 274 direct_link=self.direct_link, 275 n_clusters=self.n_clusters, 276 cluster_encode=self.cluster_encode, 277 type_clust=self.type_clust, 278 type_scaling=self.type_scaling, 279 col_sample=self.col_sample, 280 row_sample=self.row_sample, 281 seed=self.seed, 282 backend=self.backend, 283 ), 284 ), 285 ] 286 ) 287 else: 288 pipe = Pipeline( 289 steps=[ 290 ("preprocessor", preprocessor), 291 ( 292 "regressor", 293 CustomRegressor( 294 obj=model(), 295 n_hidden_features=self.n_hidden_features, 296 activation_name=self.activation_name, 297 a=self.a, 298 nodes_sim=self.nodes_sim, 299 bias=self.bias, 300 dropout=self.dropout, 301 direct_link=self.direct_link, 302 n_clusters=self.n_clusters, 303 cluster_encode=self.cluster_encode, 304 type_clust=self.type_clust, 305 type_scaling=self.type_scaling, 306 col_sample=self.col_sample, 307 row_sample=self.row_sample, 308 seed=self.seed, 309 backend=self.backend, 310 ), 311 ), 312 ] 313 ) 314 315 pipe.fit(X_train, y_train) 316 self.models[name] = pipe 317 y_pred = pipe.predict(X_test) 318 319 r_squared = r2_score(y_test, y_pred) 320 adj_rsquared = adjusted_rsquared( 321 r_squared, X_test.shape[0], X_test.shape[1] 322 ) 323 rmse = mean_squared_error(y_test, y_pred, squared=False) 324 325 names.append(name) 326 R2.append(r_squared) 327 ADJR2.append(adj_rsquared) 328 RMSE.append(rmse) 329 TIME.append(time.time() - start) 330 331 if self.custom_metric: 332 custom_metric = self.custom_metric(y_test, y_pred) 333 CUSTOM_METRIC.append(custom_metric) 334 335 if self.verbose > 0: 336 scores_verbose = { 337 "Model": name, 338 "R-Squared": r_squared, 339 "Adjusted R-Squared": adj_rsquared, 340 "RMSE": rmse, 341 "Time taken": time.time() - start, 342 } 343 344 if self.custom_metric: 345 scores_verbose[self.custom_metric.__name__] = ( 346 custom_metric 347 ) 348 349 print(scores_verbose) 350 if self.predictions: 351 predictions[name] = y_pred 352 except Exception as exception: 353 if self.ignore_warnings is False: 354 print(name + " model failed to execute") 355 print(exception) 356 357 else: 358 for name, model in tqdm(self.regressors): # do parallel exec 359 start = time.time() 360 try: 361 if "random_state" in model().get_params().keys(): 362 pipe = CustomRegressor( 363 obj=model(random_state=self.random_state), 364 n_hidden_features=self.n_hidden_features, 365 activation_name=self.activation_name, 366 a=self.a, 367 nodes_sim=self.nodes_sim, 368 bias=self.bias, 369 dropout=self.dropout, 370 direct_link=self.direct_link, 371 n_clusters=self.n_clusters, 372 cluster_encode=self.cluster_encode, 373 type_clust=self.type_clust, 374 type_scaling=self.type_scaling, 375 col_sample=self.col_sample, 376 row_sample=self.row_sample, 377 seed=self.seed, 378 backend=self.backend, 379 ) 380 else: 381 pipe = CustomRegressor( 382 obj=model(), 383 n_hidden_features=self.n_hidden_features, 384 activation_name=self.activation_name, 385 a=self.a, 386 nodes_sim=self.nodes_sim, 387 bias=self.bias, 388 dropout=self.dropout, 389 direct_link=self.direct_link, 390 n_clusters=self.n_clusters, 391 cluster_encode=self.cluster_encode, 392 type_clust=self.type_clust, 393 type_scaling=self.type_scaling, 394 col_sample=self.col_sample, 395 row_sample=self.row_sample, 396 seed=self.seed, 397 backend=self.backend, 398 ) 399 400 pipe.fit(X_train, y_train) 401 self.models[name] = pipe 402 y_pred = pipe.predict(X_test) 403 404 r_squared = r2_score(y_test, y_pred) 405 adj_rsquared = adjusted_rsquared( 406 r_squared, X_test.shape[0], X_test.shape[1] 407 ) 408 rmse = mean_squared_error(y_test, y_pred, squared=False) 409 410 names.append(name) 411 R2.append(r_squared) 412 ADJR2.append(adj_rsquared) 413 RMSE.append(rmse) 414 TIME.append(time.time() - start) 415 416 if self.custom_metric: 417 custom_metric = self.custom_metric(y_test, y_pred) 418 CUSTOM_METRIC.append(custom_metric) 419 420 if self.verbose > 0: 421 scores_verbose = { 422 "Model": name, 423 "R-Squared": r_squared, 424 "Adjusted R-Squared": adj_rsquared, 425 "RMSE": rmse, 426 "Time taken": time.time() - start, 427 } 428 429 if self.custom_metric: 430 scores_verbose[self.custom_metric.__name__] = ( 431 custom_metric 432 ) 433 434 print(scores_verbose) 435 if self.predictions: 436 predictions[name] = y_pred 437 except Exception as exception: 438 if self.ignore_warnings is False: 439 print(name + " model failed to execute") 440 print(exception) 441 442 scores = { 443 "Model": names, 444 "Adjusted R-Squared": ADJR2, 445 "R-Squared": R2, 446 "RMSE": RMSE, 447 "Time Taken": TIME, 448 } 449 450 if self.custom_metric: 451 scores[self.custom_metric.__name__] = CUSTOM_METRIC 452 453 scores = pd.DataFrame(scores) 454 scores = scores.sort_values(by="RMSE", ascending=True).set_index( 455 "Model" 456 ) 457 458 if self.predictions: 459 predictions_df = pd.DataFrame.from_dict(predictions) 460 return scores, predictions_df if self.predictions is True else scores
Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.
462 def provide_models(self, X_train, X_test, y_train, y_test): 463 """ 464 This function returns all the model objects trained in fit function. 465 If fit is not called already, then we call fit and then return the models. 466 Parameters 467 ---------- 468 X_train : array-like, 469 Training vectors, where rows is the number of samples 470 and columns is the number of features. 471 X_test : array-like, 472 Testing vectors, where rows is the number of samples 473 and columns is the number of features. 474 y_train : array-like, 475 Training vectors, where rows is the number of samples 476 and columns is the number of features. 477 y_test : array-like, 478 Testing vectors, where rows is the number of samples 479 and columns is the number of features. 480 Returns 481 ------- 482 models: dict-object, 483 Returns a dictionary with each model pipeline as value 484 with key as name of models. 485 """ 486 if len(self.models.keys()) == 0: 487 self.fit(X_train, X_test, y_train, y_test) 488 489 return self.models
This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.custom.custom.Custom
- obj
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
- sklearn.base.RegressorMixin
- score
90class LazyMTS(MTS): 91 """ 92 This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's MTS 93 Parameters 94 ---------- 95 verbose : int, optional (default=0) 96 For the liblinear and lbfgs solvers set verbose to any positive 97 number for verbosity. 98 ignore_warnings : bool, optional (default=True) 99 When set to True, the warning related to algorigms that are not able to run are ignored. 100 custom_metric : function, optional (default=None) 101 When function is provided, models are evaluated based on the custom evaluation metric provided. 102 prediction : bool, optional (default=False) 103 When set to True, the predictions of all the models models are returned as dataframe. 104 regressors : list, optional (default="all") 105 When function is provided, trains the chosen regressor(s). 106 107 Examples 108 -------- 109 110 """ 111 112 def __init__( 113 self, 114 verbose=0, 115 ignore_warnings=True, 116 custom_metric=None, 117 predictions=False, 118 random_state=42, 119 regressors="all", 120 preprocess=False, 121 # MTS attributes 122 obj=None, 123 n_hidden_features=5, 124 activation_name="relu", 125 a=0.01, 126 nodes_sim="sobol", 127 bias=True, 128 dropout=0, 129 direct_link=True, 130 n_clusters=2, 131 cluster_encode=True, 132 type_clust="kmeans", 133 type_scaling=("std", "std", "std"), 134 lags=1, 135 replications=None, 136 kernel=None, 137 agg="mean", 138 seed=123, 139 backend="cpu", 140 show_progress=False, 141 ): 142 self.verbose = verbose 143 self.ignore_warnings = ignore_warnings 144 self.custom_metric = custom_metric 145 self.predictions = predictions 146 self.models = {} 147 self.random_state = random_state 148 self.regressors = regressors 149 self.preprocess = preprocess 150 super().__init__( 151 obj=obj, 152 n_hidden_features=n_hidden_features, 153 activation_name=activation_name, 154 a=a, 155 nodes_sim=nodes_sim, 156 bias=bias, 157 dropout=dropout, 158 direct_link=direct_link, 159 n_clusters=n_clusters, 160 cluster_encode=cluster_encode, 161 type_clust=type_clust, 162 type_scaling=type_scaling, 163 seed=seed, 164 backend=backend, 165 lags=lags, 166 replications=replications, 167 kernel=kernel, 168 agg=agg, 169 show_progress=show_progress, 170 ) 171 172 def fit(self, X_train, X_test, xreg=None, new_xreg=None, **kwargs): 173 """Fit Regression algorithms to X_train, predict and score on X_test. 174 Parameters 175 ---------- 176 X_train : array-like, 177 Training vectors, where rows is the number of samples 178 and columns is the number of features. 179 X_test : array-like, 180 Testing vectors, where rows is the number of samples 181 and columns is the number of features. 182 Returns 183 ------- 184 scores : Pandas DataFrame 185 Returns metrics of all the models in a Pandas DataFrame. 186 predictions : Pandas DataFrame 187 Returns predictions of all the models in a Pandas DataFrame. 188 """ 189 R2 = [] 190 ADJR2 = [] 191 ME = [] 192 MPL = [] 193 RMSE = [] 194 MAE = [] 195 MPE = [] 196 MAPE = [] 197 198 # WIN = [] 199 names = [] 200 TIME = [] 201 predictions = {} 202 203 if self.custom_metric: 204 CUSTOM_METRIC = [] 205 206 if isinstance(X_train, np.ndarray): 207 X_train = pd.DataFrame(X_train) 208 X_test = pd.DataFrame(X_test) 209 210 X_train = convert_df_to_numeric(X_train) 211 X_test = convert_df_to_numeric(X_test) 212 213 numeric_features = X_train.select_dtypes(include=[np.number]).columns 214 categorical_features = X_train.select_dtypes(include=["object"]).columns 215 216 categorical_low, categorical_high = get_card_split( 217 X_train, categorical_features 218 ) 219 220 if self.preprocess: 221 preprocessor = ColumnTransformer( 222 transformers=[ 223 ("numeric", numeric_transformer, numeric_features), 224 ( 225 "categorical_low", 226 categorical_transformer_low, 227 categorical_low, 228 ), 229 ( 230 "categorical_high", 231 categorical_transformer_high, 232 categorical_high, 233 ), 234 ] 235 ) 236 237 if self.regressors == "all": 238 self.regressors = REGRESSORSMTS 239 else: 240 try: 241 temp_list = [] 242 for regressor in self.regressors: 243 full_name = (regressor.__name__, regressor) 244 temp_list.append(full_name) 245 self.regressors = temp_list 246 except Exception as exception: 247 print(exception) 248 print("Invalid Regressor(s)") 249 250 if self.preprocess is True: 251 for name, model in tqdm(self.regressors): # do parallel exec 252 start = time.time() 253 try: 254 if "random_state" in model().get_params().keys(): 255 pipe = Pipeline( 256 steps=[ 257 ("preprocessor", preprocessor), 258 ( 259 "regressor", 260 MTS( 261 obj=model( 262 random_state=self.random_state, 263 **kwargs 264 ), 265 n_hidden_features=self.n_hidden_features, 266 activation_name=self.activation_name, 267 a=self.a, 268 nodes_sim=self.nodes_sim, 269 bias=self.bias, 270 dropout=self.dropout, 271 direct_link=self.direct_link, 272 n_clusters=self.n_clusters, 273 cluster_encode=self.cluster_encode, 274 type_clust=self.type_clust, 275 type_scaling=self.type_scaling, 276 lags=self.lags, 277 replications=self.replications, 278 kernel=self.kernel, 279 agg=self.agg, 280 seed=self.seed, 281 backend=self.backend, 282 show_progress=self.show_progress, 283 ), 284 ), 285 ] 286 ) 287 else: 288 pipe = Pipeline( 289 steps=[ 290 ("preprocessor", preprocessor), 291 ( 292 "regressor", 293 MTS( 294 obj=model(**kwargs), 295 n_hidden_features=self.n_hidden_features, 296 activation_name=self.activation_name, 297 a=self.a, 298 nodes_sim=self.nodes_sim, 299 bias=self.bias, 300 dropout=self.dropout, 301 direct_link=self.direct_link, 302 n_clusters=self.n_clusters, 303 cluster_encode=self.cluster_encode, 304 type_clust=self.type_clust, 305 type_scaling=self.type_scaling, 306 lags=self.lags, 307 replications=self.replications, 308 kernel=self.kernel, 309 agg=self.agg, 310 seed=self.seed, 311 backend=self.backend, 312 show_progress=self.show_progress, 313 ), 314 ), 315 ] 316 ) 317 318 pipe.fit(X_train, **kwargs) 319 # pipe.fit(X_train, xreg=xreg) 320 321 self.models[name] = pipe 322 if xreg is not None: 323 assert ( 324 new_xreg is not None 325 ), "xreg and new_xreg must be provided" 326 # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) 327 X_pred = pipe["regressor"].predict( 328 h=X_test.shape[0], **kwargs 329 ) 330 rmse = mean_squared_error(X_test, X_pred, squared=False) 331 mae = mean_absolute_error(X_test, X_pred) 332 mpl = mean_pinball_loss(X_test, X_pred) 333 334 names.append(name) 335 RMSE.append(rmse) 336 MAE.append(mae) 337 MPL.append(mpl) 338 TIME.append(time.time() - start) 339 340 if self.custom_metric: 341 custom_metric = self.custom_metric(X_test, X_pred) 342 CUSTOM_METRIC.append(custom_metric) 343 344 if self.verbose > 0: 345 scores_verbose = { 346 "Model": name, 347 # "R-Squared": r_squared, 348 # "Adjusted R-Squared": adj_rsquared, 349 "RMSE": rmse, 350 "MAE": mae, 351 "MPL": mpl, 352 # "MPE": mpe, 353 # "MAPE": mape, 354 "Time taken": time.time() - start, 355 } 356 357 if self.custom_metric: 358 scores_verbose[self.custom_metric.__name__] = ( 359 custom_metric 360 ) 361 362 print(scores_verbose) 363 if self.predictions: 364 predictions[name] = X_pred 365 except Exception as exception: 366 if self.ignore_warnings is False: 367 print(name + " model failed to execute") 368 print(exception) 369 370 else: # no preprocessing 371 for name, model in tqdm(self.regressors): # do parallel exec 372 start = time.time() 373 try: 374 if "random_state" in model().get_params().keys(): 375 pipe = MTS( 376 obj=model(random_state=self.random_state, **kwargs), 377 n_hidden_features=self.n_hidden_features, 378 activation_name=self.activation_name, 379 a=self.a, 380 nodes_sim=self.nodes_sim, 381 bias=self.bias, 382 dropout=self.dropout, 383 direct_link=self.direct_link, 384 n_clusters=self.n_clusters, 385 cluster_encode=self.cluster_encode, 386 type_clust=self.type_clust, 387 type_scaling=self.type_scaling, 388 lags=self.lags, 389 replications=self.replications, 390 kernel=self.kernel, 391 agg=self.agg, 392 seed=self.seed, 393 backend=self.backend, 394 show_progress=self.show_progress, 395 ) 396 else: 397 pipe = MTS( 398 obj=model(**kwargs), 399 n_hidden_features=self.n_hidden_features, 400 activation_name=self.activation_name, 401 a=self.a, 402 nodes_sim=self.nodes_sim, 403 bias=self.bias, 404 dropout=self.dropout, 405 direct_link=self.direct_link, 406 n_clusters=self.n_clusters, 407 cluster_encode=self.cluster_encode, 408 type_clust=self.type_clust, 409 type_scaling=self.type_scaling, 410 lags=self.lags, 411 replications=self.replications, 412 kernel=self.kernel, 413 agg=self.agg, 414 seed=self.seed, 415 backend=self.backend, 416 show_progress=self.show_progress, 417 ) 418 419 pipe.fit(X_train, **kwargs) 420 # pipe.fit(X_train, xreg=xreg) # DO xreg like in `ahead` 421 422 self.models[name] = pipe 423 if xreg is not None: 424 assert ( 425 new_xreg is not None 426 ), "xreg and new_xreg must be provided" 427 428 if self.preprocess is True: 429 X_pred = pipe["regressor"].predict( 430 h=X_test.shape[0], **kwargs 431 ) 432 else: 433 X_pred = pipe.predict( 434 h=X_test.shape[0], **kwargs 435 ) # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) ## DO xreg like in `ahead` 436 437 rmse = mean_squared_error(X_test, X_pred, squared=False) 438 mae = mean_absolute_error(X_test, X_pred) 439 mpl = mean_pinball_loss(X_test, X_pred) 440 441 names.append(name) 442 RMSE.append(rmse) 443 MAE.append(mae) 444 MPL.append(mpl) 445 TIME.append(time.time() - start) 446 447 if self.custom_metric: 448 custom_metric = self.custom_metric(X_test, X_pred) 449 CUSTOM_METRIC.append(custom_metric) 450 451 if self.verbose > 0: 452 scores_verbose = { 453 "Model": name, 454 # "R-Squared": r_squared, 455 # "Adjusted R-Squared": adj_rsquared, 456 "RMSE": rmse, 457 "MAE": mae, 458 "MPL": mpl, 459 # "MPE": mpe, 460 # "MAPE": mape, 461 "Time taken": time.time() - start, 462 } 463 464 if self.custom_metric: 465 scores_verbose[self.custom_metric.__name__] = ( 466 custom_metric 467 ) 468 469 print(scores_verbose) 470 if self.predictions: 471 predictions[name] = X_pred 472 except Exception as exception: 473 if self.ignore_warnings is False: 474 print(name + " model failed to execute") 475 print(exception) 476 477 scores = { 478 "Model": names, 479 # "Adjusted R-Squared": ADJR2, 480 # "R-Squared": R2, 481 "RMSE": RMSE, 482 "MAE": MAE, 483 "MPL": MPL, 484 # "MPE": MPE, 485 # "MAPE": MAPE, 486 "Time Taken": TIME, 487 } 488 489 if self.custom_metric: 490 scores[self.custom_metric.__name__] = CUSTOM_METRIC 491 492 scores = pd.DataFrame(scores) 493 scores = scores.sort_values(by="RMSE", ascending=True).set_index( 494 "Model" 495 ) 496 497 if self.predictions: 498 predictions_df = pd.DataFrame.from_dict(predictions) 499 return scores, predictions_df if self.predictions is True else scores 500 501 def provide_models(self, X_train, X_test): 502 """ 503 This function returns all the model objects trained in fit function. 504 If fit is not called already, then we call fit and then return the models. 505 Parameters 506 ---------- 507 X_train : array-like, 508 Training vectors, where rows is the number of samples 509 and columns is the number of features. 510 X_test : array-like, 511 Testing vectors, where rows is the number of samples 512 and columns is the number of features. 513 Returns 514 ------- 515 models: dict-object, 516 Returns a dictionary with each model pipeline as value 517 with key as name of models. 518 """ 519 if len(self.models.keys()) == 0: 520 self.fit(X_train, X_test) 521 522 return self.models
This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's MTS
Parameters
verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. prediction : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. regressors : list, optional (default="all") When function is provided, trains the chosen regressor(s).
Examples
112 def __init__( 113 self, 114 verbose=0, 115 ignore_warnings=True, 116 custom_metric=None, 117 predictions=False, 118 random_state=42, 119 regressors="all", 120 preprocess=False, 121 # MTS attributes 122 obj=None, 123 n_hidden_features=5, 124 activation_name="relu", 125 a=0.01, 126 nodes_sim="sobol", 127 bias=True, 128 dropout=0, 129 direct_link=True, 130 n_clusters=2, 131 cluster_encode=True, 132 type_clust="kmeans", 133 type_scaling=("std", "std", "std"), 134 lags=1, 135 replications=None, 136 kernel=None, 137 agg="mean", 138 seed=123, 139 backend="cpu", 140 show_progress=False, 141 ): 142 self.verbose = verbose 143 self.ignore_warnings = ignore_warnings 144 self.custom_metric = custom_metric 145 self.predictions = predictions 146 self.models = {} 147 self.random_state = random_state 148 self.regressors = regressors 149 self.preprocess = preprocess 150 super().__init__( 151 obj=obj, 152 n_hidden_features=n_hidden_features, 153 activation_name=activation_name, 154 a=a, 155 nodes_sim=nodes_sim, 156 bias=bias, 157 dropout=dropout, 158 direct_link=direct_link, 159 n_clusters=n_clusters, 160 cluster_encode=cluster_encode, 161 type_clust=type_clust, 162 type_scaling=type_scaling, 163 seed=seed, 164 backend=backend, 165 lags=lags, 166 replications=replications, 167 kernel=kernel, 168 agg=agg, 169 show_progress=show_progress, 170 )
172 def fit(self, X_train, X_test, xreg=None, new_xreg=None, **kwargs): 173 """Fit Regression algorithms to X_train, predict and score on X_test. 174 Parameters 175 ---------- 176 X_train : array-like, 177 Training vectors, where rows is the number of samples 178 and columns is the number of features. 179 X_test : array-like, 180 Testing vectors, where rows is the number of samples 181 and columns is the number of features. 182 Returns 183 ------- 184 scores : Pandas DataFrame 185 Returns metrics of all the models in a Pandas DataFrame. 186 predictions : Pandas DataFrame 187 Returns predictions of all the models in a Pandas DataFrame. 188 """ 189 R2 = [] 190 ADJR2 = [] 191 ME = [] 192 MPL = [] 193 RMSE = [] 194 MAE = [] 195 MPE = [] 196 MAPE = [] 197 198 # WIN = [] 199 names = [] 200 TIME = [] 201 predictions = {} 202 203 if self.custom_metric: 204 CUSTOM_METRIC = [] 205 206 if isinstance(X_train, np.ndarray): 207 X_train = pd.DataFrame(X_train) 208 X_test = pd.DataFrame(X_test) 209 210 X_train = convert_df_to_numeric(X_train) 211 X_test = convert_df_to_numeric(X_test) 212 213 numeric_features = X_train.select_dtypes(include=[np.number]).columns 214 categorical_features = X_train.select_dtypes(include=["object"]).columns 215 216 categorical_low, categorical_high = get_card_split( 217 X_train, categorical_features 218 ) 219 220 if self.preprocess: 221 preprocessor = ColumnTransformer( 222 transformers=[ 223 ("numeric", numeric_transformer, numeric_features), 224 ( 225 "categorical_low", 226 categorical_transformer_low, 227 categorical_low, 228 ), 229 ( 230 "categorical_high", 231 categorical_transformer_high, 232 categorical_high, 233 ), 234 ] 235 ) 236 237 if self.regressors == "all": 238 self.regressors = REGRESSORSMTS 239 else: 240 try: 241 temp_list = [] 242 for regressor in self.regressors: 243 full_name = (regressor.__name__, regressor) 244 temp_list.append(full_name) 245 self.regressors = temp_list 246 except Exception as exception: 247 print(exception) 248 print("Invalid Regressor(s)") 249 250 if self.preprocess is True: 251 for name, model in tqdm(self.regressors): # do parallel exec 252 start = time.time() 253 try: 254 if "random_state" in model().get_params().keys(): 255 pipe = Pipeline( 256 steps=[ 257 ("preprocessor", preprocessor), 258 ( 259 "regressor", 260 MTS( 261 obj=model( 262 random_state=self.random_state, 263 **kwargs 264 ), 265 n_hidden_features=self.n_hidden_features, 266 activation_name=self.activation_name, 267 a=self.a, 268 nodes_sim=self.nodes_sim, 269 bias=self.bias, 270 dropout=self.dropout, 271 direct_link=self.direct_link, 272 n_clusters=self.n_clusters, 273 cluster_encode=self.cluster_encode, 274 type_clust=self.type_clust, 275 type_scaling=self.type_scaling, 276 lags=self.lags, 277 replications=self.replications, 278 kernel=self.kernel, 279 agg=self.agg, 280 seed=self.seed, 281 backend=self.backend, 282 show_progress=self.show_progress, 283 ), 284 ), 285 ] 286 ) 287 else: 288 pipe = Pipeline( 289 steps=[ 290 ("preprocessor", preprocessor), 291 ( 292 "regressor", 293 MTS( 294 obj=model(**kwargs), 295 n_hidden_features=self.n_hidden_features, 296 activation_name=self.activation_name, 297 a=self.a, 298 nodes_sim=self.nodes_sim, 299 bias=self.bias, 300 dropout=self.dropout, 301 direct_link=self.direct_link, 302 n_clusters=self.n_clusters, 303 cluster_encode=self.cluster_encode, 304 type_clust=self.type_clust, 305 type_scaling=self.type_scaling, 306 lags=self.lags, 307 replications=self.replications, 308 kernel=self.kernel, 309 agg=self.agg, 310 seed=self.seed, 311 backend=self.backend, 312 show_progress=self.show_progress, 313 ), 314 ), 315 ] 316 ) 317 318 pipe.fit(X_train, **kwargs) 319 # pipe.fit(X_train, xreg=xreg) 320 321 self.models[name] = pipe 322 if xreg is not None: 323 assert ( 324 new_xreg is not None 325 ), "xreg and new_xreg must be provided" 326 # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) 327 X_pred = pipe["regressor"].predict( 328 h=X_test.shape[0], **kwargs 329 ) 330 rmse = mean_squared_error(X_test, X_pred, squared=False) 331 mae = mean_absolute_error(X_test, X_pred) 332 mpl = mean_pinball_loss(X_test, X_pred) 333 334 names.append(name) 335 RMSE.append(rmse) 336 MAE.append(mae) 337 MPL.append(mpl) 338 TIME.append(time.time() - start) 339 340 if self.custom_metric: 341 custom_metric = self.custom_metric(X_test, X_pred) 342 CUSTOM_METRIC.append(custom_metric) 343 344 if self.verbose > 0: 345 scores_verbose = { 346 "Model": name, 347 # "R-Squared": r_squared, 348 # "Adjusted R-Squared": adj_rsquared, 349 "RMSE": rmse, 350 "MAE": mae, 351 "MPL": mpl, 352 # "MPE": mpe, 353 # "MAPE": mape, 354 "Time taken": time.time() - start, 355 } 356 357 if self.custom_metric: 358 scores_verbose[self.custom_metric.__name__] = ( 359 custom_metric 360 ) 361 362 print(scores_verbose) 363 if self.predictions: 364 predictions[name] = X_pred 365 except Exception as exception: 366 if self.ignore_warnings is False: 367 print(name + " model failed to execute") 368 print(exception) 369 370 else: # no preprocessing 371 for name, model in tqdm(self.regressors): # do parallel exec 372 start = time.time() 373 try: 374 if "random_state" in model().get_params().keys(): 375 pipe = MTS( 376 obj=model(random_state=self.random_state, **kwargs), 377 n_hidden_features=self.n_hidden_features, 378 activation_name=self.activation_name, 379 a=self.a, 380 nodes_sim=self.nodes_sim, 381 bias=self.bias, 382 dropout=self.dropout, 383 direct_link=self.direct_link, 384 n_clusters=self.n_clusters, 385 cluster_encode=self.cluster_encode, 386 type_clust=self.type_clust, 387 type_scaling=self.type_scaling, 388 lags=self.lags, 389 replications=self.replications, 390 kernel=self.kernel, 391 agg=self.agg, 392 seed=self.seed, 393 backend=self.backend, 394 show_progress=self.show_progress, 395 ) 396 else: 397 pipe = MTS( 398 obj=model(**kwargs), 399 n_hidden_features=self.n_hidden_features, 400 activation_name=self.activation_name, 401 a=self.a, 402 nodes_sim=self.nodes_sim, 403 bias=self.bias, 404 dropout=self.dropout, 405 direct_link=self.direct_link, 406 n_clusters=self.n_clusters, 407 cluster_encode=self.cluster_encode, 408 type_clust=self.type_clust, 409 type_scaling=self.type_scaling, 410 lags=self.lags, 411 replications=self.replications, 412 kernel=self.kernel, 413 agg=self.agg, 414 seed=self.seed, 415 backend=self.backend, 416 show_progress=self.show_progress, 417 ) 418 419 pipe.fit(X_train, **kwargs) 420 # pipe.fit(X_train, xreg=xreg) # DO xreg like in `ahead` 421 422 self.models[name] = pipe 423 if xreg is not None: 424 assert ( 425 new_xreg is not None 426 ), "xreg and new_xreg must be provided" 427 428 if self.preprocess is True: 429 X_pred = pipe["regressor"].predict( 430 h=X_test.shape[0], **kwargs 431 ) 432 else: 433 X_pred = pipe.predict( 434 h=X_test.shape[0], **kwargs 435 ) # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) ## DO xreg like in `ahead` 436 437 rmse = mean_squared_error(X_test, X_pred, squared=False) 438 mae = mean_absolute_error(X_test, X_pred) 439 mpl = mean_pinball_loss(X_test, X_pred) 440 441 names.append(name) 442 RMSE.append(rmse) 443 MAE.append(mae) 444 MPL.append(mpl) 445 TIME.append(time.time() - start) 446 447 if self.custom_metric: 448 custom_metric = self.custom_metric(X_test, X_pred) 449 CUSTOM_METRIC.append(custom_metric) 450 451 if self.verbose > 0: 452 scores_verbose = { 453 "Model": name, 454 # "R-Squared": r_squared, 455 # "Adjusted R-Squared": adj_rsquared, 456 "RMSE": rmse, 457 "MAE": mae, 458 "MPL": mpl, 459 # "MPE": mpe, 460 # "MAPE": mape, 461 "Time taken": time.time() - start, 462 } 463 464 if self.custom_metric: 465 scores_verbose[self.custom_metric.__name__] = ( 466 custom_metric 467 ) 468 469 print(scores_verbose) 470 if self.predictions: 471 predictions[name] = X_pred 472 except Exception as exception: 473 if self.ignore_warnings is False: 474 print(name + " model failed to execute") 475 print(exception) 476 477 scores = { 478 "Model": names, 479 # "Adjusted R-Squared": ADJR2, 480 # "R-Squared": R2, 481 "RMSE": RMSE, 482 "MAE": MAE, 483 "MPL": MPL, 484 # "MPE": MPE, 485 # "MAPE": MAPE, 486 "Time Taken": TIME, 487 } 488 489 if self.custom_metric: 490 scores[self.custom_metric.__name__] = CUSTOM_METRIC 491 492 scores = pd.DataFrame(scores) 493 scores = scores.sort_values(by="RMSE", ascending=True).set_index( 494 "Model" 495 ) 496 497 if self.predictions: 498 predictions_df = pd.DataFrame.from_dict(predictions) 499 return scores, predictions_df if self.predictions is True else scores
Fit Regression algorithms to X_train, predict and score on X_test.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.
501 def provide_models(self, X_train, X_test): 502 """ 503 This function returns all the model objects trained in fit function. 504 If fit is not called already, then we call fit and then return the models. 505 Parameters 506 ---------- 507 X_train : array-like, 508 Training vectors, where rows is the number of samples 509 and columns is the number of features. 510 X_test : array-like, 511 Testing vectors, where rows is the number of samples 512 and columns is the number of features. 513 Returns 514 ------- 515 models: dict-object, 516 Returns a dictionary with each model pipeline as value 517 with key as name of models. 518 """ 519 if len(self.models.keys()) == 0: 520 self.fit(X_train, X_test) 521 522 return self.models
This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- MTS
- obj
- n_series
- lags
- type_pi
- replications
- kernel
- agg
- show_progress
- series_names
- input_dates
- fit_objs_
- y_
- X_
- xreg_
- y_means_
- mean_
- upper_
- lower_
- output_dates_
- preds_std_
- alpha_
- return_std_
- df_
- residuals_
- residuals_sims_
- kde_
- sims_
- predict
- score
- plot
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
81class LazyDeepClassifier(Custom, ClassifierMixin): 82 """ 83 This module helps in fitting to all the classification algorithms that are available in Scikit-learn to nnetsauce's CustomClassifier 84 Parameters 85 ---------- 86 verbose : int, optional (default=0) 87 For the liblinear and lbfgs solvers set verbose to any positive 88 number for verbosity. 89 ignore_warnings : bool, optional (default=True) 90 When set to True, the warning related to algorigms that are not able to run are ignored. 91 custom_metric : function, optional (default=None) 92 When function is provided, models are evaluated based on the custom evaluation metric provided. 93 prediction : bool, optional (default=False) 94 When set to True, the predictions of all the models models are returned as dataframe. 95 classifiers : list, optional (default="all") 96 When function is provided, trains the chosen classifier(s). 97 n_jobs : int, when possible, run in parallel 98 99 Examples 100 -------- 101 >>> import nnetsauce as ns 102 >>> from sklearn.datasets import load_breast_cancer 103 >>> from sklearn.model_selection import train_test_split 104 >>> data = load_breast_cancer() 105 >>> X = data.data 106 >>> y= data.target 107 >>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123) 108 >>> clf = ns.LazyDeepClassifier(verbose=0, ignore_warnings=True, custom_metric=None) 109 >>> models, predictions = clf.fit(X_train, X_test, y_train, y_test) 110 >>> model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test) 111 >>> print(models) 112 """ 113 114 def __init__( 115 self, 116 verbose=0, 117 ignore_warnings=True, 118 custom_metric=None, 119 predictions=False, 120 random_state=42, 121 classifiers="all", 122 n_jobs=None, 123 # Defining depth 124 n_layers=3, 125 # CustomClassifier attributes 126 obj=None, 127 n_hidden_features=5, 128 activation_name="relu", 129 a=0.01, 130 nodes_sim="sobol", 131 bias=True, 132 dropout=0, 133 direct_link=True, 134 n_clusters=2, 135 cluster_encode=True, 136 type_clust="kmeans", 137 type_scaling=("std", "std", "std"), 138 col_sample=1, 139 row_sample=1, 140 seed=123, 141 backend="cpu", 142 ): 143 self.verbose = verbose 144 self.ignore_warnings = ignore_warnings 145 self.custom_metric = custom_metric 146 self.predictions = predictions 147 self.models = {} 148 self.random_state = random_state 149 self.classifiers = classifiers 150 self.n_layers = n_layers - 1 151 self.n_jobs = n_jobs 152 super().__init__( 153 obj=obj, 154 n_hidden_features=n_hidden_features, 155 activation_name=activation_name, 156 a=a, 157 nodes_sim=nodes_sim, 158 bias=bias, 159 dropout=dropout, 160 direct_link=direct_link, 161 n_clusters=n_clusters, 162 cluster_encode=cluster_encode, 163 type_clust=type_clust, 164 type_scaling=type_scaling, 165 col_sample=col_sample, 166 row_sample=row_sample, 167 seed=seed, 168 backend=backend, 169 ) 170 171 def fit(self, X_train, X_test, y_train, y_test): 172 """Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test. 173 Parameters 174 ---------- 175 X_train : array-like, 176 Training vectors, where rows is the number of samples 177 and columns is the number of features. 178 X_test : array-like, 179 Testing vectors, where rows is the number of samples 180 and columns is the number of features. 181 y_train : array-like, 182 Training vectors, where rows is the number of samples 183 and columns is the number of features. 184 y_test : array-like, 185 Testing vectors, where rows is the number of samples 186 and columns is the number of features. 187 Returns 188 ------- 189 scores : Pandas DataFrame 190 Returns metrics of all the models in a Pandas DataFrame. 191 predictions : Pandas DataFrame 192 Returns predictions of all the models in a Pandas DataFrame. 193 """ 194 Accuracy = [] 195 B_Accuracy = [] 196 ROC_AUC = [] 197 F1 = [] 198 names = [] 199 TIME = [] 200 predictions = {} 201 202 if self.custom_metric is not None: 203 CUSTOM_METRIC = [] 204 205 if isinstance(X_train, np.ndarray): 206 X_train = pd.DataFrame(X_train) 207 X_test = pd.DataFrame(X_test) 208 209 numeric_features = X_train.select_dtypes(include=[np.number]).columns 210 categorical_features = X_train.select_dtypes(include=["object"]).columns 211 212 categorical_low, categorical_high = get_card_split( 213 X_train, categorical_features 214 ) 215 216 if self.classifiers == "all": 217 self.classifiers = [ 218 item 219 for sublist in [ 220 CLASSIFIERS, 221 MULTITASKCLASSIFIERS, 222 SIMPLEMULTITASKCLASSIFIERS, 223 ] 224 for item in sublist 225 ] 226 else: 227 try: 228 temp_list = [] 229 for classifier in self.classifiers: 230 full_name = (classifier.__name__, classifier) 231 temp_list.append(full_name) 232 self.classifiers = temp_list 233 except Exception as exception: 234 print(exception) 235 print("Invalid Classifier(s)") 236 237 for name, model in tqdm(self.classifiers): # do parallel exec 238 start = time.time() 239 try: 240 if "random_state" in model().get_params().keys(): 241 layer_clf = CustomClassifier( 242 obj=model(random_state=self.random_state), 243 n_hidden_features=self.n_hidden_features, 244 activation_name=self.activation_name, 245 a=self.a, 246 nodes_sim=self.nodes_sim, 247 bias=self.bias, 248 dropout=self.dropout, 249 direct_link=self.direct_link, 250 n_clusters=self.n_clusters, 251 cluster_encode=self.cluster_encode, 252 type_clust=self.type_clust, 253 type_scaling=self.type_scaling, 254 col_sample=self.col_sample, 255 row_sample=self.row_sample, 256 seed=self.seed, 257 backend=self.backend, 258 ) 259 260 else: 261 layer_clf = CustomClassifier( 262 obj=model(), 263 n_hidden_features=self.n_hidden_features, 264 activation_name=self.activation_name, 265 a=self.a, 266 nodes_sim=self.nodes_sim, 267 bias=self.bias, 268 dropout=self.dropout, 269 direct_link=self.direct_link, 270 n_clusters=self.n_clusters, 271 cluster_encode=self.cluster_encode, 272 type_clust=self.type_clust, 273 type_scaling=self.type_scaling, 274 col_sample=self.col_sample, 275 row_sample=self.row_sample, 276 seed=self.seed, 277 backend=self.backend, 278 ) 279 280 layer_clf.fit(X_train, y_train) 281 282 for _ in range(self.n_layers): 283 layer_clf = deepcopy( 284 CustomClassifier( 285 obj=layer_clf, 286 n_hidden_features=self.n_hidden_features, 287 activation_name=self.activation_name, 288 a=self.a, 289 nodes_sim=self.nodes_sim, 290 bias=self.bias, 291 dropout=self.dropout, 292 direct_link=self.direct_link, 293 n_clusters=self.n_clusters, 294 cluster_encode=self.cluster_encode, 295 type_clust=self.type_clust, 296 type_scaling=self.type_scaling, 297 col_sample=self.col_sample, 298 row_sample=self.row_sample, 299 seed=self.seed, 300 backend=self.backend, 301 ) 302 ) 303 304 # layer_clf.fit(X_train, y_train) 305 306 layer_clf.fit(X_train, y_train) 307 308 self.models[name] = layer_clf 309 y_pred = layer_clf.predict(X_test) 310 accuracy = accuracy_score(y_test, y_pred, normalize=True) 311 b_accuracy = balanced_accuracy_score(y_test, y_pred) 312 f1 = f1_score(y_test, y_pred, average="weighted") 313 try: 314 roc_auc = roc_auc_score(y_test, y_pred) 315 except Exception as exception: 316 roc_auc = None 317 if self.ignore_warnings is False: 318 print("ROC AUC couldn't be calculated for " + name) 319 print(exception) 320 names.append(name) 321 Accuracy.append(accuracy) 322 B_Accuracy.append(b_accuracy) 323 ROC_AUC.append(roc_auc) 324 F1.append(f1) 325 TIME.append(time.time() - start) 326 if self.custom_metric is not None: 327 custom_metric = self.custom_metric(y_test, y_pred) 328 CUSTOM_METRIC.append(custom_metric) 329 if self.verbose > 0: 330 if self.custom_metric is not None: 331 print( 332 { 333 "Model": name, 334 "Accuracy": accuracy, 335 "Balanced Accuracy": b_accuracy, 336 "ROC AUC": roc_auc, 337 "F1 Score": f1, 338 self.custom_metric.__name__: custom_metric, 339 "Time taken": time.time() - start, 340 } 341 ) 342 else: 343 print( 344 { 345 "Model": name, 346 "Accuracy": accuracy, 347 "Balanced Accuracy": b_accuracy, 348 "ROC AUC": roc_auc, 349 "F1 Score": f1, 350 "Time taken": time.time() - start, 351 } 352 ) 353 if self.predictions: 354 predictions[name] = y_pred 355 except Exception as exception: 356 if self.ignore_warnings is False: 357 print(name + " model failed to execute") 358 print(exception) 359 360 if self.custom_metric is None: 361 scores = pd.DataFrame( 362 { 363 "Model": names, 364 "Accuracy": Accuracy, 365 "Balanced Accuracy": B_Accuracy, 366 "ROC AUC": ROC_AUC, 367 "F1 Score": F1, 368 "Time Taken": TIME, 369 } 370 ) 371 else: 372 scores = pd.DataFrame( 373 { 374 "Model": names, 375 "Accuracy": Accuracy, 376 "Balanced Accuracy": B_Accuracy, 377 "ROC AUC": ROC_AUC, 378 "F1 Score": F1, 379 self.custom_metric.__name__: CUSTOM_METRIC, 380 "Time Taken": TIME, 381 } 382 ) 383 scores = scores.sort_values(by="Accuracy", ascending=False).set_index( 384 "Model" 385 ) 386 387 if self.predictions: 388 predictions_df = pd.DataFrame.from_dict(predictions) 389 return scores, predictions_df if self.predictions is True else scores 390 391 def provide_models(self, X_train, X_test, y_train, y_test): 392 """ 393 This function returns all the model objects trained in fit function. 394 If fit is not called already, then we call fit and then return the models. 395 Parameters 396 ---------- 397 X_train : array-like, 398 Training vectors, where rows is the number of samples 399 and columns is the number of features. 400 X_test : array-like, 401 Testing vectors, where rows is the number of samples 402 and columns is the number of features. 403 y_train : array-like, 404 Training vectors, where rows is the number of samples 405 and columns is the number of features. 406 y_test : array-like, 407 Testing vectors, where rows is the number of samples 408 and columns is the number of features. 409 Returns 410 ------- 411 models: dict-object, 412 Returns a dictionary with each model pipeline as value 413 with key as name of models. 414 """ 415 if len(self.models.keys()) == 0: 416 self.fit(X_train, X_test, y_train, y_test) 417 418 return self.models
This module helps in fitting to all the classification algorithms that are available in Scikit-learn to nnetsauce's CustomClassifier
Parameters
verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. prediction : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. classifiers : list, optional (default="all") When function is provided, trains the chosen classifier(s). n_jobs : int, when possible, run in parallel
Examples
>>> import nnetsauce as ns
>>> from sklearn.datasets import load_breast_cancer
>>> from sklearn.model_selection import train_test_split
>>> data = load_breast_cancer()
>>> X = data.data
>>> y= data.target
>>> X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2, random_state=123)
>>> clf = ns.LazyDeepClassifier(verbose=0, ignore_warnings=True, custom_metric=None)
>>> models, predictions = clf.fit(X_train, X_test, y_train, y_test)
>>> model_dictionary = clf.provide_models(X_train,X_test,y_train,y_test)
>>> print(models)
114 def __init__( 115 self, 116 verbose=0, 117 ignore_warnings=True, 118 custom_metric=None, 119 predictions=False, 120 random_state=42, 121 classifiers="all", 122 n_jobs=None, 123 # Defining depth 124 n_layers=3, 125 # CustomClassifier attributes 126 obj=None, 127 n_hidden_features=5, 128 activation_name="relu", 129 a=0.01, 130 nodes_sim="sobol", 131 bias=True, 132 dropout=0, 133 direct_link=True, 134 n_clusters=2, 135 cluster_encode=True, 136 type_clust="kmeans", 137 type_scaling=("std", "std", "std"), 138 col_sample=1, 139 row_sample=1, 140 seed=123, 141 backend="cpu", 142 ): 143 self.verbose = verbose 144 self.ignore_warnings = ignore_warnings 145 self.custom_metric = custom_metric 146 self.predictions = predictions 147 self.models = {} 148 self.random_state = random_state 149 self.classifiers = classifiers 150 self.n_layers = n_layers - 1 151 self.n_jobs = n_jobs 152 super().__init__( 153 obj=obj, 154 n_hidden_features=n_hidden_features, 155 activation_name=activation_name, 156 a=a, 157 nodes_sim=nodes_sim, 158 bias=bias, 159 dropout=dropout, 160 direct_link=direct_link, 161 n_clusters=n_clusters, 162 cluster_encode=cluster_encode, 163 type_clust=type_clust, 164 type_scaling=type_scaling, 165 col_sample=col_sample, 166 row_sample=row_sample, 167 seed=seed, 168 backend=backend, 169 )
171 def fit(self, X_train, X_test, y_train, y_test): 172 """Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test. 173 Parameters 174 ---------- 175 X_train : array-like, 176 Training vectors, where rows is the number of samples 177 and columns is the number of features. 178 X_test : array-like, 179 Testing vectors, where rows is the number of samples 180 and columns is the number of features. 181 y_train : array-like, 182 Training vectors, where rows is the number of samples 183 and columns is the number of features. 184 y_test : array-like, 185 Testing vectors, where rows is the number of samples 186 and columns is the number of features. 187 Returns 188 ------- 189 scores : Pandas DataFrame 190 Returns metrics of all the models in a Pandas DataFrame. 191 predictions : Pandas DataFrame 192 Returns predictions of all the models in a Pandas DataFrame. 193 """ 194 Accuracy = [] 195 B_Accuracy = [] 196 ROC_AUC = [] 197 F1 = [] 198 names = [] 199 TIME = [] 200 predictions = {} 201 202 if self.custom_metric is not None: 203 CUSTOM_METRIC = [] 204 205 if isinstance(X_train, np.ndarray): 206 X_train = pd.DataFrame(X_train) 207 X_test = pd.DataFrame(X_test) 208 209 numeric_features = X_train.select_dtypes(include=[np.number]).columns 210 categorical_features = X_train.select_dtypes(include=["object"]).columns 211 212 categorical_low, categorical_high = get_card_split( 213 X_train, categorical_features 214 ) 215 216 if self.classifiers == "all": 217 self.classifiers = [ 218 item 219 for sublist in [ 220 CLASSIFIERS, 221 MULTITASKCLASSIFIERS, 222 SIMPLEMULTITASKCLASSIFIERS, 223 ] 224 for item in sublist 225 ] 226 else: 227 try: 228 temp_list = [] 229 for classifier in self.classifiers: 230 full_name = (classifier.__name__, classifier) 231 temp_list.append(full_name) 232 self.classifiers = temp_list 233 except Exception as exception: 234 print(exception) 235 print("Invalid Classifier(s)") 236 237 for name, model in tqdm(self.classifiers): # do parallel exec 238 start = time.time() 239 try: 240 if "random_state" in model().get_params().keys(): 241 layer_clf = CustomClassifier( 242 obj=model(random_state=self.random_state), 243 n_hidden_features=self.n_hidden_features, 244 activation_name=self.activation_name, 245 a=self.a, 246 nodes_sim=self.nodes_sim, 247 bias=self.bias, 248 dropout=self.dropout, 249 direct_link=self.direct_link, 250 n_clusters=self.n_clusters, 251 cluster_encode=self.cluster_encode, 252 type_clust=self.type_clust, 253 type_scaling=self.type_scaling, 254 col_sample=self.col_sample, 255 row_sample=self.row_sample, 256 seed=self.seed, 257 backend=self.backend, 258 ) 259 260 else: 261 layer_clf = CustomClassifier( 262 obj=model(), 263 n_hidden_features=self.n_hidden_features, 264 activation_name=self.activation_name, 265 a=self.a, 266 nodes_sim=self.nodes_sim, 267 bias=self.bias, 268 dropout=self.dropout, 269 direct_link=self.direct_link, 270 n_clusters=self.n_clusters, 271 cluster_encode=self.cluster_encode, 272 type_clust=self.type_clust, 273 type_scaling=self.type_scaling, 274 col_sample=self.col_sample, 275 row_sample=self.row_sample, 276 seed=self.seed, 277 backend=self.backend, 278 ) 279 280 layer_clf.fit(X_train, y_train) 281 282 for _ in range(self.n_layers): 283 layer_clf = deepcopy( 284 CustomClassifier( 285 obj=layer_clf, 286 n_hidden_features=self.n_hidden_features, 287 activation_name=self.activation_name, 288 a=self.a, 289 nodes_sim=self.nodes_sim, 290 bias=self.bias, 291 dropout=self.dropout, 292 direct_link=self.direct_link, 293 n_clusters=self.n_clusters, 294 cluster_encode=self.cluster_encode, 295 type_clust=self.type_clust, 296 type_scaling=self.type_scaling, 297 col_sample=self.col_sample, 298 row_sample=self.row_sample, 299 seed=self.seed, 300 backend=self.backend, 301 ) 302 ) 303 304 # layer_clf.fit(X_train, y_train) 305 306 layer_clf.fit(X_train, y_train) 307 308 self.models[name] = layer_clf 309 y_pred = layer_clf.predict(X_test) 310 accuracy = accuracy_score(y_test, y_pred, normalize=True) 311 b_accuracy = balanced_accuracy_score(y_test, y_pred) 312 f1 = f1_score(y_test, y_pred, average="weighted") 313 try: 314 roc_auc = roc_auc_score(y_test, y_pred) 315 except Exception as exception: 316 roc_auc = None 317 if self.ignore_warnings is False: 318 print("ROC AUC couldn't be calculated for " + name) 319 print(exception) 320 names.append(name) 321 Accuracy.append(accuracy) 322 B_Accuracy.append(b_accuracy) 323 ROC_AUC.append(roc_auc) 324 F1.append(f1) 325 TIME.append(time.time() - start) 326 if self.custom_metric is not None: 327 custom_metric = self.custom_metric(y_test, y_pred) 328 CUSTOM_METRIC.append(custom_metric) 329 if self.verbose > 0: 330 if self.custom_metric is not None: 331 print( 332 { 333 "Model": name, 334 "Accuracy": accuracy, 335 "Balanced Accuracy": b_accuracy, 336 "ROC AUC": roc_auc, 337 "F1 Score": f1, 338 self.custom_metric.__name__: custom_metric, 339 "Time taken": time.time() - start, 340 } 341 ) 342 else: 343 print( 344 { 345 "Model": name, 346 "Accuracy": accuracy, 347 "Balanced Accuracy": b_accuracy, 348 "ROC AUC": roc_auc, 349 "F1 Score": f1, 350 "Time taken": time.time() - start, 351 } 352 ) 353 if self.predictions: 354 predictions[name] = y_pred 355 except Exception as exception: 356 if self.ignore_warnings is False: 357 print(name + " model failed to execute") 358 print(exception) 359 360 if self.custom_metric is None: 361 scores = pd.DataFrame( 362 { 363 "Model": names, 364 "Accuracy": Accuracy, 365 "Balanced Accuracy": B_Accuracy, 366 "ROC AUC": ROC_AUC, 367 "F1 Score": F1, 368 "Time Taken": TIME, 369 } 370 ) 371 else: 372 scores = pd.DataFrame( 373 { 374 "Model": names, 375 "Accuracy": Accuracy, 376 "Balanced Accuracy": B_Accuracy, 377 "ROC AUC": ROC_AUC, 378 "F1 Score": F1, 379 self.custom_metric.__name__: CUSTOM_METRIC, 380 "Time Taken": TIME, 381 } 382 ) 383 scores = scores.sort_values(by="Accuracy", ascending=False).set_index( 384 "Model" 385 ) 386 387 if self.predictions: 388 predictions_df = pd.DataFrame.from_dict(predictions) 389 return scores, predictions_df if self.predictions is True else scores
Fit Classification algorithms to X_train and y_train, predict and score on X_test, y_test.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.
391 def provide_models(self, X_train, X_test, y_train, y_test): 392 """ 393 This function returns all the model objects trained in fit function. 394 If fit is not called already, then we call fit and then return the models. 395 Parameters 396 ---------- 397 X_train : array-like, 398 Training vectors, where rows is the number of samples 399 and columns is the number of features. 400 X_test : array-like, 401 Testing vectors, where rows is the number of samples 402 and columns is the number of features. 403 y_train : array-like, 404 Training vectors, where rows is the number of samples 405 and columns is the number of features. 406 y_test : array-like, 407 Testing vectors, where rows is the number of samples 408 and columns is the number of features. 409 Returns 410 ------- 411 models: dict-object, 412 Returns a dictionary with each model pipeline as value 413 with key as name of models. 414 """ 415 if len(self.models.keys()) == 0: 416 self.fit(X_train, X_test, y_train, y_test) 417 418 return self.models
This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.custom.custom.Custom
- obj
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
- sklearn.base.ClassifierMixin
- score
85class LazyDeepRegressor(Custom, RegressorMixin): 86 """ 87 This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's CustomRegressor 88 Parameters 89 ---------- 90 verbose : int, optional (default=0) 91 For the liblinear and lbfgs solvers set verbose to any positive 92 number for verbosity. 93 ignore_warnings : bool, optional (default=True) 94 When set to True, the warning related to algorigms that are not able to run are ignored. 95 custom_metric : function, optional (default=None) 96 When function is provided, models are evaluated based on the custom evaluation metric provided. 97 prediction : bool, optional (default=False) 98 When set to True, the predictions of all the models models are returned as dataframe. 99 regressors : list, optional (default="all") 100 When function is provided, trains the chosen regressor(s). 101 n_jobs : int, when possible, run in parallel 102 103 Examples 104 -------- 105 >>> from lazypredict.Supervised import LazyRegressor 106 >>> from sklearn import datasets 107 >>> from sklearn.utils import shuffle 108 >>> import numpy as np 109 110 >>> diabetes = datasets.load_diabetes() 111 >>> X, y = shuffle(diabetes.data, diabetes.target, random_state=13) 112 >>> X = X.astype(np.float32) 113 114 >>> offset = int(X.shape[0] * 0.9) 115 >>> X_train, y_train = X[:offset], y[:offset] 116 >>> X_test, y_test = X[offset:], y[offset:] 117 118 >>> reg = LazyDeepRegressor(verbose=0, ignore_warnings=False, custom_metric=None) 119 >>> models, predictions = reg.fit(X_train, X_test, y_train, y_test) 120 >>> model_dictionary = reg.provide_models(X_train, X_test, y_train, y_test) 121 >>> print(models) 122 """ 123 124 def __init__( 125 self, 126 verbose=0, 127 ignore_warnings=True, 128 custom_metric=None, 129 predictions=False, 130 random_state=42, 131 regressors="all", 132 n_jobs=None, 133 # Defining depth 134 n_layers=3, 135 # CustomRegressor attributes 136 obj=None, 137 n_hidden_features=5, 138 activation_name="relu", 139 a=0.01, 140 nodes_sim="sobol", 141 bias=True, 142 dropout=0, 143 direct_link=True, 144 n_clusters=2, 145 cluster_encode=True, 146 type_clust="kmeans", 147 type_scaling=("std", "std", "std"), 148 col_sample=1, 149 row_sample=1, 150 seed=123, 151 backend="cpu", 152 ): 153 self.verbose = verbose 154 self.ignore_warnings = ignore_warnings 155 self.custom_metric = custom_metric 156 self.predictions = predictions 157 self.models = {} 158 self.random_state = random_state 159 self.regressors = regressors 160 self.n_layers = n_layers - 1 161 self.n_jobs = n_jobs 162 super().__init__( 163 obj=obj, 164 n_hidden_features=n_hidden_features, 165 activation_name=activation_name, 166 a=a, 167 nodes_sim=nodes_sim, 168 bias=bias, 169 dropout=dropout, 170 direct_link=direct_link, 171 n_clusters=n_clusters, 172 cluster_encode=cluster_encode, 173 type_clust=type_clust, 174 type_scaling=type_scaling, 175 col_sample=col_sample, 176 row_sample=row_sample, 177 seed=seed, 178 backend=backend, 179 ) 180 181 def fit(self, X_train, X_test, y_train, y_test): 182 """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test. 183 Parameters 184 ---------- 185 X_train : array-like, 186 Training vectors, where rows is the number of samples 187 and columns is the number of features. 188 X_test : array-like, 189 Testing vectors, where rows is the number of samples 190 and columns is the number of features. 191 y_train : array-like, 192 Training vectors, where rows is the number of samples 193 and columns is the number of features. 194 y_test : array-like, 195 Testing vectors, where rows is the number of samples 196 and columns is the number of features. 197 Returns 198 ------- 199 scores : Pandas DataFrame 200 Returns metrics of all the models in a Pandas DataFrame. 201 predictions : Pandas DataFrame 202 Returns predictions of all the models in a Pandas DataFrame. 203 """ 204 R2 = [] 205 ADJR2 = [] 206 RMSE = [] 207 # WIN = [] 208 names = [] 209 TIME = [] 210 predictions = {} 211 212 if self.custom_metric: 213 CUSTOM_METRIC = [] 214 215 if isinstance(X_train, np.ndarray): 216 X_train = pd.DataFrame(X_train) 217 X_test = pd.DataFrame(X_test) 218 219 numeric_features = X_train.select_dtypes(include=[np.number]).columns 220 categorical_features = X_train.select_dtypes(include=["object"]).columns 221 222 categorical_low, categorical_high = get_card_split( 223 X_train, categorical_features 224 ) 225 226 if self.regressors == "all": 227 self.regressors = REGRESSORS 228 else: 229 try: 230 temp_list = [] 231 for regressor in self.regressors: 232 full_name = (regressor.__name__, regressor) 233 temp_list.append(full_name) 234 self.regressors = temp_list 235 except Exception as exception: 236 print(exception) 237 print("Invalid Regressor(s)") 238 239 for name, model in tqdm(self.regressors): # do parallel exec 240 start = time.time() 241 try: 242 if "random_state" in model().get_params().keys(): 243 layer_regr = CustomRegressor( 244 obj=model(random_state=self.random_state), 245 n_hidden_features=self.n_hidden_features, 246 activation_name=self.activation_name, 247 a=self.a, 248 nodes_sim=self.nodes_sim, 249 bias=self.bias, 250 dropout=self.dropout, 251 direct_link=self.direct_link, 252 n_clusters=self.n_clusters, 253 cluster_encode=self.cluster_encode, 254 type_clust=self.type_clust, 255 type_scaling=self.type_scaling, 256 col_sample=self.col_sample, 257 row_sample=self.row_sample, 258 seed=self.seed, 259 backend=self.backend, 260 ) 261 else: 262 layer_regr = CustomRegressor( 263 obj=model(), 264 n_hidden_features=self.n_hidden_features, 265 activation_name=self.activation_name, 266 a=self.a, 267 nodes_sim=self.nodes_sim, 268 bias=self.bias, 269 dropout=self.dropout, 270 direct_link=self.direct_link, 271 n_clusters=self.n_clusters, 272 cluster_encode=self.cluster_encode, 273 type_clust=self.type_clust, 274 type_scaling=self.type_scaling, 275 col_sample=self.col_sample, 276 row_sample=self.row_sample, 277 seed=self.seed, 278 backend=self.backend, 279 ) 280 281 layer_regr.fit(X_train, y_train) 282 283 for _ in range(self.n_layers): 284 layer_regr = deepcopy( 285 CustomRegressor( 286 obj=layer_regr, 287 n_hidden_features=self.n_hidden_features, 288 activation_name=self.activation_name, 289 a=self.a, 290 nodes_sim=self.nodes_sim, 291 bias=self.bias, 292 dropout=self.dropout, 293 direct_link=self.direct_link, 294 n_clusters=self.n_clusters, 295 cluster_encode=self.cluster_encode, 296 type_clust=self.type_clust, 297 type_scaling=self.type_scaling, 298 col_sample=self.col_sample, 299 row_sample=self.row_sample, 300 seed=self.seed, 301 backend=self.backend, 302 ) 303 ) 304 305 # layer_regr.fit(X_train, y_train) 306 307 layer_regr.fit(X_train, y_train) 308 309 self.models[name] = layer_regr 310 y_pred = layer_regr.predict(X_test) 311 312 r_squared = r2_score(y_test, y_pred) 313 adj_rsquared = adjusted_rsquared( 314 r_squared, X_test.shape[0], X_test.shape[1] 315 ) 316 rmse = mean_squared_error(y_test, y_pred, squared=False) 317 318 names.append(name) 319 R2.append(r_squared) 320 ADJR2.append(adj_rsquared) 321 RMSE.append(rmse) 322 TIME.append(time.time() - start) 323 324 if self.custom_metric: 325 custom_metric = self.custom_metric(y_test, y_pred) 326 CUSTOM_METRIC.append(custom_metric) 327 328 if self.verbose > 0: 329 scores_verbose = { 330 "Model": name, 331 "R-Squared": r_squared, 332 "Adjusted R-Squared": adj_rsquared, 333 "RMSE": rmse, 334 "Time taken": time.time() - start, 335 } 336 337 if self.custom_metric: 338 scores_verbose[self.custom_metric.__name__] = ( 339 custom_metric 340 ) 341 342 print(scores_verbose) 343 if self.predictions: 344 predictions[name] = y_pred 345 except Exception as exception: 346 if self.ignore_warnings is False: 347 print(name + " model failed to execute") 348 print(exception) 349 350 scores = { 351 "Model": names, 352 "Adjusted R-Squared": ADJR2, 353 "R-Squared": R2, 354 "RMSE": RMSE, 355 "Time Taken": TIME, 356 } 357 358 if self.custom_metric: 359 scores[self.custom_metric.__name__] = CUSTOM_METRIC 360 361 scores = pd.DataFrame(scores) 362 scores = scores.sort_values(by="RMSE", ascending=True).set_index( 363 "Model" 364 ) 365 366 if self.predictions: 367 predictions_df = pd.DataFrame.from_dict(predictions) 368 return scores, predictions_df if self.predictions is True else scores 369 370 def provide_models(self, X_train, X_test, y_train, y_test): 371 """ 372 This function returns all the model objects trained in fit function. 373 If fit is not called already, then we call fit and then return the models. 374 Parameters 375 ---------- 376 X_train : array-like, 377 Training vectors, where rows is the number of samples 378 and columns is the number of features. 379 X_test : array-like, 380 Testing vectors, where rows is the number of samples 381 and columns is the number of features. 382 y_train : array-like, 383 Training vectors, where rows is the number of samples 384 and columns is the number of features. 385 y_test : array-like, 386 Testing vectors, where rows is the number of samples 387 and columns is the number of features. 388 Returns 389 ------- 390 models: dict-object, 391 Returns a dictionary with each model pipeline as value 392 with key as name of models. 393 """ 394 if len(self.models.keys()) == 0: 395 self.fit(X_train, X_test, y_train, y_test) 396 397 return self.models
This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's CustomRegressor
Parameters
verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. prediction : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. regressors : list, optional (default="all") When function is provided, trains the chosen regressor(s). n_jobs : int, when possible, run in parallel
Examples
>>> from lazypredict.Supervised import LazyRegressor
>>> from sklearn import datasets
>>> from sklearn.utils import shuffle
>>> import numpy as np
>>> diabetes = datasets.load_diabetes()
>>> X, y = shuffle(diabetes.data, diabetes.target, random_state=13)
>>> X = X.astype(np.float32)
>>> offset = int(X.shape[0] * 0.9)
>>> X_train, y_train = X[:offset], y[:offset]
>>> X_test, y_test = X[offset:], y[offset:]
>>> reg = LazyDeepRegressor(verbose=0, ignore_warnings=False, custom_metric=None)
>>> models, predictions = reg.fit(X_train, X_test, y_train, y_test)
>>> model_dictionary = reg.provide_models(X_train, X_test, y_train, y_test)
>>> print(models)
124 def __init__( 125 self, 126 verbose=0, 127 ignore_warnings=True, 128 custom_metric=None, 129 predictions=False, 130 random_state=42, 131 regressors="all", 132 n_jobs=None, 133 # Defining depth 134 n_layers=3, 135 # CustomRegressor attributes 136 obj=None, 137 n_hidden_features=5, 138 activation_name="relu", 139 a=0.01, 140 nodes_sim="sobol", 141 bias=True, 142 dropout=0, 143 direct_link=True, 144 n_clusters=2, 145 cluster_encode=True, 146 type_clust="kmeans", 147 type_scaling=("std", "std", "std"), 148 col_sample=1, 149 row_sample=1, 150 seed=123, 151 backend="cpu", 152 ): 153 self.verbose = verbose 154 self.ignore_warnings = ignore_warnings 155 self.custom_metric = custom_metric 156 self.predictions = predictions 157 self.models = {} 158 self.random_state = random_state 159 self.regressors = regressors 160 self.n_layers = n_layers - 1 161 self.n_jobs = n_jobs 162 super().__init__( 163 obj=obj, 164 n_hidden_features=n_hidden_features, 165 activation_name=activation_name, 166 a=a, 167 nodes_sim=nodes_sim, 168 bias=bias, 169 dropout=dropout, 170 direct_link=direct_link, 171 n_clusters=n_clusters, 172 cluster_encode=cluster_encode, 173 type_clust=type_clust, 174 type_scaling=type_scaling, 175 col_sample=col_sample, 176 row_sample=row_sample, 177 seed=seed, 178 backend=backend, 179 )
181 def fit(self, X_train, X_test, y_train, y_test): 182 """Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test. 183 Parameters 184 ---------- 185 X_train : array-like, 186 Training vectors, where rows is the number of samples 187 and columns is the number of features. 188 X_test : array-like, 189 Testing vectors, where rows is the number of samples 190 and columns is the number of features. 191 y_train : array-like, 192 Training vectors, where rows is the number of samples 193 and columns is the number of features. 194 y_test : array-like, 195 Testing vectors, where rows is the number of samples 196 and columns is the number of features. 197 Returns 198 ------- 199 scores : Pandas DataFrame 200 Returns metrics of all the models in a Pandas DataFrame. 201 predictions : Pandas DataFrame 202 Returns predictions of all the models in a Pandas DataFrame. 203 """ 204 R2 = [] 205 ADJR2 = [] 206 RMSE = [] 207 # WIN = [] 208 names = [] 209 TIME = [] 210 predictions = {} 211 212 if self.custom_metric: 213 CUSTOM_METRIC = [] 214 215 if isinstance(X_train, np.ndarray): 216 X_train = pd.DataFrame(X_train) 217 X_test = pd.DataFrame(X_test) 218 219 numeric_features = X_train.select_dtypes(include=[np.number]).columns 220 categorical_features = X_train.select_dtypes(include=["object"]).columns 221 222 categorical_low, categorical_high = get_card_split( 223 X_train, categorical_features 224 ) 225 226 if self.regressors == "all": 227 self.regressors = REGRESSORS 228 else: 229 try: 230 temp_list = [] 231 for regressor in self.regressors: 232 full_name = (regressor.__name__, regressor) 233 temp_list.append(full_name) 234 self.regressors = temp_list 235 except Exception as exception: 236 print(exception) 237 print("Invalid Regressor(s)") 238 239 for name, model in tqdm(self.regressors): # do parallel exec 240 start = time.time() 241 try: 242 if "random_state" in model().get_params().keys(): 243 layer_regr = CustomRegressor( 244 obj=model(random_state=self.random_state), 245 n_hidden_features=self.n_hidden_features, 246 activation_name=self.activation_name, 247 a=self.a, 248 nodes_sim=self.nodes_sim, 249 bias=self.bias, 250 dropout=self.dropout, 251 direct_link=self.direct_link, 252 n_clusters=self.n_clusters, 253 cluster_encode=self.cluster_encode, 254 type_clust=self.type_clust, 255 type_scaling=self.type_scaling, 256 col_sample=self.col_sample, 257 row_sample=self.row_sample, 258 seed=self.seed, 259 backend=self.backend, 260 ) 261 else: 262 layer_regr = CustomRegressor( 263 obj=model(), 264 n_hidden_features=self.n_hidden_features, 265 activation_name=self.activation_name, 266 a=self.a, 267 nodes_sim=self.nodes_sim, 268 bias=self.bias, 269 dropout=self.dropout, 270 direct_link=self.direct_link, 271 n_clusters=self.n_clusters, 272 cluster_encode=self.cluster_encode, 273 type_clust=self.type_clust, 274 type_scaling=self.type_scaling, 275 col_sample=self.col_sample, 276 row_sample=self.row_sample, 277 seed=self.seed, 278 backend=self.backend, 279 ) 280 281 layer_regr.fit(X_train, y_train) 282 283 for _ in range(self.n_layers): 284 layer_regr = deepcopy( 285 CustomRegressor( 286 obj=layer_regr, 287 n_hidden_features=self.n_hidden_features, 288 activation_name=self.activation_name, 289 a=self.a, 290 nodes_sim=self.nodes_sim, 291 bias=self.bias, 292 dropout=self.dropout, 293 direct_link=self.direct_link, 294 n_clusters=self.n_clusters, 295 cluster_encode=self.cluster_encode, 296 type_clust=self.type_clust, 297 type_scaling=self.type_scaling, 298 col_sample=self.col_sample, 299 row_sample=self.row_sample, 300 seed=self.seed, 301 backend=self.backend, 302 ) 303 ) 304 305 # layer_regr.fit(X_train, y_train) 306 307 layer_regr.fit(X_train, y_train) 308 309 self.models[name] = layer_regr 310 y_pred = layer_regr.predict(X_test) 311 312 r_squared = r2_score(y_test, y_pred) 313 adj_rsquared = adjusted_rsquared( 314 r_squared, X_test.shape[0], X_test.shape[1] 315 ) 316 rmse = mean_squared_error(y_test, y_pred, squared=False) 317 318 names.append(name) 319 R2.append(r_squared) 320 ADJR2.append(adj_rsquared) 321 RMSE.append(rmse) 322 TIME.append(time.time() - start) 323 324 if self.custom_metric: 325 custom_metric = self.custom_metric(y_test, y_pred) 326 CUSTOM_METRIC.append(custom_metric) 327 328 if self.verbose > 0: 329 scores_verbose = { 330 "Model": name, 331 "R-Squared": r_squared, 332 "Adjusted R-Squared": adj_rsquared, 333 "RMSE": rmse, 334 "Time taken": time.time() - start, 335 } 336 337 if self.custom_metric: 338 scores_verbose[self.custom_metric.__name__] = ( 339 custom_metric 340 ) 341 342 print(scores_verbose) 343 if self.predictions: 344 predictions[name] = y_pred 345 except Exception as exception: 346 if self.ignore_warnings is False: 347 print(name + " model failed to execute") 348 print(exception) 349 350 scores = { 351 "Model": names, 352 "Adjusted R-Squared": ADJR2, 353 "R-Squared": R2, 354 "RMSE": RMSE, 355 "Time Taken": TIME, 356 } 357 358 if self.custom_metric: 359 scores[self.custom_metric.__name__] = CUSTOM_METRIC 360 361 scores = pd.DataFrame(scores) 362 scores = scores.sort_values(by="RMSE", ascending=True).set_index( 363 "Model" 364 ) 365 366 if self.predictions: 367 predictions_df = pd.DataFrame.from_dict(predictions) 368 return scores, predictions_df if self.predictions is True else scores
Fit Regression algorithms to X_train and y_train, predict and score on X_test, y_test.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.
370 def provide_models(self, X_train, X_test, y_train, y_test): 371 """ 372 This function returns all the model objects trained in fit function. 373 If fit is not called already, then we call fit and then return the models. 374 Parameters 375 ---------- 376 X_train : array-like, 377 Training vectors, where rows is the number of samples 378 and columns is the number of features. 379 X_test : array-like, 380 Testing vectors, where rows is the number of samples 381 and columns is the number of features. 382 y_train : array-like, 383 Training vectors, where rows is the number of samples 384 and columns is the number of features. 385 y_test : array-like, 386 Testing vectors, where rows is the number of samples 387 and columns is the number of features. 388 Returns 389 ------- 390 models: dict-object, 391 Returns a dictionary with each model pipeline as value 392 with key as name of models. 393 """ 394 if len(self.models.keys()) == 0: 395 self.fit(X_train, X_test, y_train, y_test) 396 397 return self.models
This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features. y_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. y_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.custom.custom.Custom
- obj
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
- sklearn.base.RegressorMixin
- score
91class LazyDeepMTS(MTS): 92 """ 93 This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's MTS 94 Parameters 95 ---------- 96 verbose : int, optional (default=0) 97 For the liblinear and lbfgs solvers set verbose to any positive 98 number for verbosity. 99 ignore_warnings : bool, optional (default=True) 100 When set to True, the warning related to algorigms that are not able to run are ignored. 101 custom_metric : function, optional (default=None) 102 When function is provided, models are evaluated based on the custom evaluation metric provided. 103 prediction : bool, optional (default=False) 104 When set to True, the predictions of all the models models are returned as dataframe. 105 regressors : list, optional (default="all") 106 When function is provided, trains the chosen regressor(s). 107 108 Examples 109 -------- 110 111 """ 112 113 def __init__( 114 self, 115 verbose=0, 116 ignore_warnings=True, 117 custom_metric=None, 118 predictions=False, 119 random_state=42, 120 regressors="all", 121 preprocess=False, 122 # Defining depth 123 n_layers=3, 124 # MTS attributes 125 obj=None, 126 n_hidden_features=5, 127 activation_name="relu", 128 a=0.01, 129 nodes_sim="sobol", 130 bias=True, 131 dropout=0, 132 direct_link=True, 133 n_clusters=2, 134 cluster_encode=True, 135 type_clust="kmeans", 136 type_scaling=("std", "std", "std"), 137 lags=1, 138 replications=None, 139 kernel=None, 140 agg="mean", 141 seed=123, 142 backend="cpu", 143 show_progress=False, 144 ): 145 self.verbose = verbose 146 self.ignore_warnings = ignore_warnings 147 self.custom_metric = custom_metric 148 self.predictions = predictions 149 self.models = {} 150 self.random_state = random_state 151 self.regressors = regressors 152 self.preprocess = preprocess 153 self.n_layers = n_layers 154 super().__init__( 155 obj=obj, 156 n_hidden_features=n_hidden_features, 157 activation_name=activation_name, 158 a=a, 159 nodes_sim=nodes_sim, 160 bias=bias, 161 dropout=dropout, 162 direct_link=direct_link, 163 n_clusters=n_clusters, 164 cluster_encode=cluster_encode, 165 type_clust=type_clust, 166 type_scaling=type_scaling, 167 seed=seed, 168 backend=backend, 169 lags=lags, 170 replications=replications, 171 kernel=kernel, 172 agg=agg, 173 show_progress=show_progress, 174 ) 175 176 def fit(self, X_train, X_test, xreg=None, new_xreg=None, **kwargs): 177 """Fit Regression algorithms to X_train, predict and score on X_test. 178 Parameters 179 ---------- 180 X_train : array-like, 181 Training vectors, where rows is the number of samples 182 and columns is the number of features. 183 X_test : array-like, 184 Testing vectors, where rows is the number of samples 185 and columns is the number of features. 186 Returns 187 ------- 188 scores : Pandas DataFrame 189 Returns metrics of all the models in a Pandas DataFrame. 190 predictions : Pandas DataFrame 191 Returns predictions of all the models in a Pandas DataFrame. 192 """ 193 R2 = [] 194 ADJR2 = [] 195 ME = [] 196 MPL = [] 197 RMSE = [] 198 MAE = [] 199 MPE = [] 200 MAPE = [] 201 202 # WIN = [] 203 names = [] 204 TIME = [] 205 predictions = {} 206 207 if self.custom_metric: 208 CUSTOM_METRIC = [] 209 210 if isinstance(X_train, np.ndarray): 211 X_train = pd.DataFrame(X_train) 212 X_test = pd.DataFrame(X_test) 213 214 X_train = convert_df_to_numeric(X_train) 215 X_test = convert_df_to_numeric(X_test) 216 217 numeric_features = X_train.select_dtypes(include=[np.number]).columns 218 categorical_features = X_train.select_dtypes(include=["object"]).columns 219 220 categorical_low, categorical_high = get_card_split( 221 X_train, categorical_features 222 ) 223 224 if self.preprocess: 225 preprocessor = ColumnTransformer( 226 transformers=[ 227 ("numeric", numeric_transformer, numeric_features), 228 ( 229 "categorical_low", 230 categorical_transformer_low, 231 categorical_low, 232 ), 233 ( 234 "categorical_high", 235 categorical_transformer_high, 236 categorical_high, 237 ), 238 ] 239 ) 240 241 if self.regressors == "all": 242 self.regressors = REGRESSORSDEEPMTS 243 else: 244 try: 245 temp_list = [] 246 for regressor in self.regressors: 247 full_name = (regressor.__name__, regressor) 248 temp_list.append(full_name) 249 self.regressors = temp_list 250 except Exception as exception: 251 print(exception) 252 print("Invalid Regressor(s)") 253 254 if self.preprocess is True: 255 for name, model in tqdm(self.regressors): # do parallel exec 256 start = time.time() 257 try: 258 if "random_state" in model().get_params().keys(): 259 pipe = Pipeline( 260 steps=[ 261 ("preprocessor", preprocessor), 262 ( 263 "regressor", 264 DeepMTS( 265 obj=model( 266 random_state=self.random_state, 267 **kwargs 268 ), 269 n_layers=self.n_layers, 270 n_hidden_features=self.n_hidden_features, 271 activation_name=self.activation_name, 272 a=self.a, 273 nodes_sim=self.nodes_sim, 274 bias=self.bias, 275 dropout=self.dropout, 276 direct_link=self.direct_link, 277 n_clusters=self.n_clusters, 278 cluster_encode=self.cluster_encode, 279 type_clust=self.type_clust, 280 type_scaling=self.type_scaling, 281 lags=self.lags, 282 replications=self.replications, 283 kernel=self.kernel, 284 agg=self.agg, 285 seed=self.seed, 286 backend=self.backend, 287 show_progress=self.show_progress, 288 ), 289 ), 290 ] 291 ) 292 else: 293 pipe = Pipeline( 294 steps=[ 295 ("preprocessor", preprocessor), 296 ( 297 "regressor", 298 DeepMTS( 299 obj=model(**kwargs), 300 n_layers=self.n_layers, 301 n_hidden_features=self.n_hidden_features, 302 activation_name=self.activation_name, 303 a=self.a, 304 nodes_sim=self.nodes_sim, 305 bias=self.bias, 306 dropout=self.dropout, 307 direct_link=self.direct_link, 308 n_clusters=self.n_clusters, 309 cluster_encode=self.cluster_encode, 310 type_clust=self.type_clust, 311 type_scaling=self.type_scaling, 312 lags=self.lags, 313 replications=self.replications, 314 kernel=self.kernel, 315 agg=self.agg, 316 seed=self.seed, 317 backend=self.backend, 318 show_progress=self.show_progress, 319 ), 320 ), 321 ] 322 ) 323 324 pipe.fit(X_train, **kwargs) 325 # pipe.fit(X_train, xreg=xreg) 326 327 self.models[name] = pipe 328 if xreg is not None: 329 assert ( 330 new_xreg is not None 331 ), "xreg and new_xreg must be provided" 332 # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) 333 X_pred = pipe["regressor"].predict( 334 h=X_test.shape[0], **kwargs 335 ) 336 rmse = mean_squared_error(X_test, X_pred, squared=False) 337 mae = mean_absolute_error(X_test, X_pred) 338 mpl = mean_pinball_loss(X_test, X_pred) 339 340 names.append(name) 341 RMSE.append(rmse) 342 MAE.append(mae) 343 MPL.append(mpl) 344 TIME.append(time.time() - start) 345 346 if self.custom_metric: 347 custom_metric = self.custom_metric(X_test, X_pred) 348 CUSTOM_METRIC.append(custom_metric) 349 350 if self.verbose > 0: 351 scores_verbose = { 352 "Model": name, 353 # "R-Squared": r_squared, 354 # "Adjusted R-Squared": adj_rsquared, 355 "RMSE": rmse, 356 "MAE": mae, 357 "MPL": mpl, 358 # "MPE": mpe, 359 # "MAPE": mape, 360 "Time taken": time.time() - start, 361 } 362 363 if self.custom_metric: 364 scores_verbose[self.custom_metric.__name__] = ( 365 custom_metric 366 ) 367 368 print(scores_verbose) 369 if self.predictions: 370 predictions[name] = X_pred 371 except Exception as exception: 372 if self.ignore_warnings is False: 373 print(name + " model failed to execute") 374 print(exception) 375 376 else: # no preprocessing 377 for name, model in tqdm(self.regressors): # do parallel exec 378 start = time.time() 379 try: 380 if "random_state" in model().get_params().keys(): 381 pipe = DeepMTS( 382 obj=model(random_state=self.random_state, **kwargs), 383 n_hidden_features=self.n_hidden_features, 384 activation_name=self.activation_name, 385 a=self.a, 386 nodes_sim=self.nodes_sim, 387 bias=self.bias, 388 dropout=self.dropout, 389 direct_link=self.direct_link, 390 n_clusters=self.n_clusters, 391 cluster_encode=self.cluster_encode, 392 type_clust=self.type_clust, 393 type_scaling=self.type_scaling, 394 lags=self.lags, 395 replications=self.replications, 396 kernel=self.kernel, 397 agg=self.agg, 398 seed=self.seed, 399 backend=self.backend, 400 show_progress=self.show_progress, 401 ) 402 else: 403 pipe = DeepMTS( 404 obj=model(**kwargs), 405 n_hidden_features=self.n_hidden_features, 406 activation_name=self.activation_name, 407 a=self.a, 408 nodes_sim=self.nodes_sim, 409 bias=self.bias, 410 dropout=self.dropout, 411 direct_link=self.direct_link, 412 n_clusters=self.n_clusters, 413 cluster_encode=self.cluster_encode, 414 type_clust=self.type_clust, 415 type_scaling=self.type_scaling, 416 lags=self.lags, 417 replications=self.replications, 418 kernel=self.kernel, 419 agg=self.agg, 420 seed=self.seed, 421 backend=self.backend, 422 show_progress=self.show_progress, 423 ) 424 425 pipe.fit(X_train, **kwargs) 426 # pipe.fit(X_train, xreg=xreg) # DO xreg like in `ahead` 427 428 self.models[name] = pipe 429 if xreg is not None: 430 assert ( 431 new_xreg is not None 432 ), "xreg and new_xreg must be provided" 433 434 if self.preprocess is True: 435 X_pred = pipe["regressor"].predict( 436 h=X_test.shape[0], **kwargs 437 ) 438 else: 439 X_pred = pipe.predict( 440 h=X_test.shape[0], **kwargs 441 ) # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) ## DO xreg like in `ahead` 442 443 rmse = mean_squared_error(X_test, X_pred, squared=False) 444 mae = mean_absolute_error(X_test, X_pred) 445 mpl = mean_pinball_loss(X_test, X_pred) 446 447 names.append(name) 448 RMSE.append(rmse) 449 MAE.append(mae) 450 MPL.append(mpl) 451 TIME.append(time.time() - start) 452 453 if self.custom_metric: 454 custom_metric = self.custom_metric(X_test, X_pred) 455 CUSTOM_METRIC.append(custom_metric) 456 457 if self.verbose > 0: 458 scores_verbose = { 459 "Model": name, 460 # "R-Squared": r_squared, 461 # "Adjusted R-Squared": adj_rsquared, 462 "RMSE": rmse, 463 "MAE": mae, 464 "MPL": mpl, 465 # "MPE": mpe, 466 # "MAPE": mape, 467 "Time taken": time.time() - start, 468 } 469 470 if self.custom_metric: 471 scores_verbose[self.custom_metric.__name__] = ( 472 custom_metric 473 ) 474 475 print(scores_verbose) 476 if self.predictions: 477 predictions[name] = X_pred 478 except Exception as exception: 479 if self.ignore_warnings is False: 480 print(name + " model failed to execute") 481 print(exception) 482 483 scores = { 484 "Model": names, 485 # "Adjusted R-Squared": ADJR2, 486 # "R-Squared": R2, 487 "RMSE": RMSE, 488 "MAE": MAE, 489 "MPL": MPL, 490 # "MPE": MPE, 491 # "MAPE": MAPE, 492 "Time Taken": TIME, 493 } 494 495 if self.custom_metric: 496 scores[self.custom_metric.__name__] = CUSTOM_METRIC 497 498 scores = pd.DataFrame(scores) 499 scores = scores.sort_values(by="RMSE", ascending=True).set_index( 500 "Model" 501 ) 502 503 if self.predictions: 504 predictions_df = pd.DataFrame.from_dict(predictions) 505 return scores, predictions_df if self.predictions is True else scores 506 507 def provide_models(self, X_train, X_test): 508 """ 509 This function returns all the model objects trained in fit function. 510 If fit is not called already, then we call fit and then return the models. 511 Parameters 512 ---------- 513 X_train : array-like, 514 Training vectors, where rows is the number of samples 515 and columns is the number of features. 516 X_test : array-like, 517 Testing vectors, where rows is the number of samples 518 and columns is the number of features. 519 Returns 520 ------- 521 models: dict-object, 522 Returns a dictionary with each model pipeline as value 523 with key as name of models. 524 """ 525 if len(self.models.keys()) == 0: 526 self.fit(X_train, X_test) 527 528 return self.models
This module helps in fitting regression models that are available in Scikit-learn to nnetsauce's MTS
Parameters
verbose : int, optional (default=0) For the liblinear and lbfgs solvers set verbose to any positive number for verbosity. ignore_warnings : bool, optional (default=True) When set to True, the warning related to algorigms that are not able to run are ignored. custom_metric : function, optional (default=None) When function is provided, models are evaluated based on the custom evaluation metric provided. prediction : bool, optional (default=False) When set to True, the predictions of all the models models are returned as dataframe. regressors : list, optional (default="all") When function is provided, trains the chosen regressor(s).
Examples
113 def __init__( 114 self, 115 verbose=0, 116 ignore_warnings=True, 117 custom_metric=None, 118 predictions=False, 119 random_state=42, 120 regressors="all", 121 preprocess=False, 122 # Defining depth 123 n_layers=3, 124 # MTS attributes 125 obj=None, 126 n_hidden_features=5, 127 activation_name="relu", 128 a=0.01, 129 nodes_sim="sobol", 130 bias=True, 131 dropout=0, 132 direct_link=True, 133 n_clusters=2, 134 cluster_encode=True, 135 type_clust="kmeans", 136 type_scaling=("std", "std", "std"), 137 lags=1, 138 replications=None, 139 kernel=None, 140 agg="mean", 141 seed=123, 142 backend="cpu", 143 show_progress=False, 144 ): 145 self.verbose = verbose 146 self.ignore_warnings = ignore_warnings 147 self.custom_metric = custom_metric 148 self.predictions = predictions 149 self.models = {} 150 self.random_state = random_state 151 self.regressors = regressors 152 self.preprocess = preprocess 153 self.n_layers = n_layers 154 super().__init__( 155 obj=obj, 156 n_hidden_features=n_hidden_features, 157 activation_name=activation_name, 158 a=a, 159 nodes_sim=nodes_sim, 160 bias=bias, 161 dropout=dropout, 162 direct_link=direct_link, 163 n_clusters=n_clusters, 164 cluster_encode=cluster_encode, 165 type_clust=type_clust, 166 type_scaling=type_scaling, 167 seed=seed, 168 backend=backend, 169 lags=lags, 170 replications=replications, 171 kernel=kernel, 172 agg=agg, 173 show_progress=show_progress, 174 )
176 def fit(self, X_train, X_test, xreg=None, new_xreg=None, **kwargs): 177 """Fit Regression algorithms to X_train, predict and score on X_test. 178 Parameters 179 ---------- 180 X_train : array-like, 181 Training vectors, where rows is the number of samples 182 and columns is the number of features. 183 X_test : array-like, 184 Testing vectors, where rows is the number of samples 185 and columns is the number of features. 186 Returns 187 ------- 188 scores : Pandas DataFrame 189 Returns metrics of all the models in a Pandas DataFrame. 190 predictions : Pandas DataFrame 191 Returns predictions of all the models in a Pandas DataFrame. 192 """ 193 R2 = [] 194 ADJR2 = [] 195 ME = [] 196 MPL = [] 197 RMSE = [] 198 MAE = [] 199 MPE = [] 200 MAPE = [] 201 202 # WIN = [] 203 names = [] 204 TIME = [] 205 predictions = {} 206 207 if self.custom_metric: 208 CUSTOM_METRIC = [] 209 210 if isinstance(X_train, np.ndarray): 211 X_train = pd.DataFrame(X_train) 212 X_test = pd.DataFrame(X_test) 213 214 X_train = convert_df_to_numeric(X_train) 215 X_test = convert_df_to_numeric(X_test) 216 217 numeric_features = X_train.select_dtypes(include=[np.number]).columns 218 categorical_features = X_train.select_dtypes(include=["object"]).columns 219 220 categorical_low, categorical_high = get_card_split( 221 X_train, categorical_features 222 ) 223 224 if self.preprocess: 225 preprocessor = ColumnTransformer( 226 transformers=[ 227 ("numeric", numeric_transformer, numeric_features), 228 ( 229 "categorical_low", 230 categorical_transformer_low, 231 categorical_low, 232 ), 233 ( 234 "categorical_high", 235 categorical_transformer_high, 236 categorical_high, 237 ), 238 ] 239 ) 240 241 if self.regressors == "all": 242 self.regressors = REGRESSORSDEEPMTS 243 else: 244 try: 245 temp_list = [] 246 for regressor in self.regressors: 247 full_name = (regressor.__name__, regressor) 248 temp_list.append(full_name) 249 self.regressors = temp_list 250 except Exception as exception: 251 print(exception) 252 print("Invalid Regressor(s)") 253 254 if self.preprocess is True: 255 for name, model in tqdm(self.regressors): # do parallel exec 256 start = time.time() 257 try: 258 if "random_state" in model().get_params().keys(): 259 pipe = Pipeline( 260 steps=[ 261 ("preprocessor", preprocessor), 262 ( 263 "regressor", 264 DeepMTS( 265 obj=model( 266 random_state=self.random_state, 267 **kwargs 268 ), 269 n_layers=self.n_layers, 270 n_hidden_features=self.n_hidden_features, 271 activation_name=self.activation_name, 272 a=self.a, 273 nodes_sim=self.nodes_sim, 274 bias=self.bias, 275 dropout=self.dropout, 276 direct_link=self.direct_link, 277 n_clusters=self.n_clusters, 278 cluster_encode=self.cluster_encode, 279 type_clust=self.type_clust, 280 type_scaling=self.type_scaling, 281 lags=self.lags, 282 replications=self.replications, 283 kernel=self.kernel, 284 agg=self.agg, 285 seed=self.seed, 286 backend=self.backend, 287 show_progress=self.show_progress, 288 ), 289 ), 290 ] 291 ) 292 else: 293 pipe = Pipeline( 294 steps=[ 295 ("preprocessor", preprocessor), 296 ( 297 "regressor", 298 DeepMTS( 299 obj=model(**kwargs), 300 n_layers=self.n_layers, 301 n_hidden_features=self.n_hidden_features, 302 activation_name=self.activation_name, 303 a=self.a, 304 nodes_sim=self.nodes_sim, 305 bias=self.bias, 306 dropout=self.dropout, 307 direct_link=self.direct_link, 308 n_clusters=self.n_clusters, 309 cluster_encode=self.cluster_encode, 310 type_clust=self.type_clust, 311 type_scaling=self.type_scaling, 312 lags=self.lags, 313 replications=self.replications, 314 kernel=self.kernel, 315 agg=self.agg, 316 seed=self.seed, 317 backend=self.backend, 318 show_progress=self.show_progress, 319 ), 320 ), 321 ] 322 ) 323 324 pipe.fit(X_train, **kwargs) 325 # pipe.fit(X_train, xreg=xreg) 326 327 self.models[name] = pipe 328 if xreg is not None: 329 assert ( 330 new_xreg is not None 331 ), "xreg and new_xreg must be provided" 332 # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) 333 X_pred = pipe["regressor"].predict( 334 h=X_test.shape[0], **kwargs 335 ) 336 rmse = mean_squared_error(X_test, X_pred, squared=False) 337 mae = mean_absolute_error(X_test, X_pred) 338 mpl = mean_pinball_loss(X_test, X_pred) 339 340 names.append(name) 341 RMSE.append(rmse) 342 MAE.append(mae) 343 MPL.append(mpl) 344 TIME.append(time.time() - start) 345 346 if self.custom_metric: 347 custom_metric = self.custom_metric(X_test, X_pred) 348 CUSTOM_METRIC.append(custom_metric) 349 350 if self.verbose > 0: 351 scores_verbose = { 352 "Model": name, 353 # "R-Squared": r_squared, 354 # "Adjusted R-Squared": adj_rsquared, 355 "RMSE": rmse, 356 "MAE": mae, 357 "MPL": mpl, 358 # "MPE": mpe, 359 # "MAPE": mape, 360 "Time taken": time.time() - start, 361 } 362 363 if self.custom_metric: 364 scores_verbose[self.custom_metric.__name__] = ( 365 custom_metric 366 ) 367 368 print(scores_verbose) 369 if self.predictions: 370 predictions[name] = X_pred 371 except Exception as exception: 372 if self.ignore_warnings is False: 373 print(name + " model failed to execute") 374 print(exception) 375 376 else: # no preprocessing 377 for name, model in tqdm(self.regressors): # do parallel exec 378 start = time.time() 379 try: 380 if "random_state" in model().get_params().keys(): 381 pipe = DeepMTS( 382 obj=model(random_state=self.random_state, **kwargs), 383 n_hidden_features=self.n_hidden_features, 384 activation_name=self.activation_name, 385 a=self.a, 386 nodes_sim=self.nodes_sim, 387 bias=self.bias, 388 dropout=self.dropout, 389 direct_link=self.direct_link, 390 n_clusters=self.n_clusters, 391 cluster_encode=self.cluster_encode, 392 type_clust=self.type_clust, 393 type_scaling=self.type_scaling, 394 lags=self.lags, 395 replications=self.replications, 396 kernel=self.kernel, 397 agg=self.agg, 398 seed=self.seed, 399 backend=self.backend, 400 show_progress=self.show_progress, 401 ) 402 else: 403 pipe = DeepMTS( 404 obj=model(**kwargs), 405 n_hidden_features=self.n_hidden_features, 406 activation_name=self.activation_name, 407 a=self.a, 408 nodes_sim=self.nodes_sim, 409 bias=self.bias, 410 dropout=self.dropout, 411 direct_link=self.direct_link, 412 n_clusters=self.n_clusters, 413 cluster_encode=self.cluster_encode, 414 type_clust=self.type_clust, 415 type_scaling=self.type_scaling, 416 lags=self.lags, 417 replications=self.replications, 418 kernel=self.kernel, 419 agg=self.agg, 420 seed=self.seed, 421 backend=self.backend, 422 show_progress=self.show_progress, 423 ) 424 425 pipe.fit(X_train, **kwargs) 426 # pipe.fit(X_train, xreg=xreg) # DO xreg like in `ahead` 427 428 self.models[name] = pipe 429 if xreg is not None: 430 assert ( 431 new_xreg is not None 432 ), "xreg and new_xreg must be provided" 433 434 if self.preprocess is True: 435 X_pred = pipe["regressor"].predict( 436 h=X_test.shape[0], **kwargs 437 ) 438 else: 439 X_pred = pipe.predict( 440 h=X_test.shape[0], **kwargs 441 ) # X_pred = pipe.predict(h=X_test.shape[0], new_xreg=new_xreg) ## DO xreg like in `ahead` 442 443 rmse = mean_squared_error(X_test, X_pred, squared=False) 444 mae = mean_absolute_error(X_test, X_pred) 445 mpl = mean_pinball_loss(X_test, X_pred) 446 447 names.append(name) 448 RMSE.append(rmse) 449 MAE.append(mae) 450 MPL.append(mpl) 451 TIME.append(time.time() - start) 452 453 if self.custom_metric: 454 custom_metric = self.custom_metric(X_test, X_pred) 455 CUSTOM_METRIC.append(custom_metric) 456 457 if self.verbose > 0: 458 scores_verbose = { 459 "Model": name, 460 # "R-Squared": r_squared, 461 # "Adjusted R-Squared": adj_rsquared, 462 "RMSE": rmse, 463 "MAE": mae, 464 "MPL": mpl, 465 # "MPE": mpe, 466 # "MAPE": mape, 467 "Time taken": time.time() - start, 468 } 469 470 if self.custom_metric: 471 scores_verbose[self.custom_metric.__name__] = ( 472 custom_metric 473 ) 474 475 print(scores_verbose) 476 if self.predictions: 477 predictions[name] = X_pred 478 except Exception as exception: 479 if self.ignore_warnings is False: 480 print(name + " model failed to execute") 481 print(exception) 482 483 scores = { 484 "Model": names, 485 # "Adjusted R-Squared": ADJR2, 486 # "R-Squared": R2, 487 "RMSE": RMSE, 488 "MAE": MAE, 489 "MPL": MPL, 490 # "MPE": MPE, 491 # "MAPE": MAPE, 492 "Time Taken": TIME, 493 } 494 495 if self.custom_metric: 496 scores[self.custom_metric.__name__] = CUSTOM_METRIC 497 498 scores = pd.DataFrame(scores) 499 scores = scores.sort_values(by="RMSE", ascending=True).set_index( 500 "Model" 501 ) 502 503 if self.predictions: 504 predictions_df = pd.DataFrame.from_dict(predictions) 505 return scores, predictions_df if self.predictions is True else scores
Fit Regression algorithms to X_train, predict and score on X_test.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
scores : Pandas DataFrame Returns metrics of all the models in a Pandas DataFrame. predictions : Pandas DataFrame Returns predictions of all the models in a Pandas DataFrame.
507 def provide_models(self, X_train, X_test): 508 """ 509 This function returns all the model objects trained in fit function. 510 If fit is not called already, then we call fit and then return the models. 511 Parameters 512 ---------- 513 X_train : array-like, 514 Training vectors, where rows is the number of samples 515 and columns is the number of features. 516 X_test : array-like, 517 Testing vectors, where rows is the number of samples 518 and columns is the number of features. 519 Returns 520 ------- 521 models: dict-object, 522 Returns a dictionary with each model pipeline as value 523 with key as name of models. 524 """ 525 if len(self.models.keys()) == 0: 526 self.fit(X_train, X_test) 527 528 return self.models
This function returns all the model objects trained in fit function. If fit is not called already, then we call fit and then return the models.
Parameters
X_train : array-like, Training vectors, where rows is the number of samples and columns is the number of features. X_test : array-like, Testing vectors, where rows is the number of samples and columns is the number of features.
Returns
models: dict-object, Returns a dictionary with each model pipeline as value with key as name of models.
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- MTS
- obj
- n_series
- lags
- type_pi
- replications
- kernel
- agg
- show_progress
- series_names
- input_dates
- fit_objs_
- y_
- X_
- xreg_
- y_means_
- mean_
- upper_
- lower_
- output_dates_
- preds_std_
- alpha_
- return_std_
- df_
- residuals_
- residuals_sims_
- kde_
- sims_
- predict
- score
- plot
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
23class MTS(Base): 24 """Univariate and multivariate time series (MTS) forecasting with Quasi-Randomized networks (Work in progress /!\) 25 26 Parameters: 27 28 obj: object. 29 any object containing a method fit (obj.fit()) and a method predict 30 (obj.predict()). 31 32 n_hidden_features: int. 33 number of nodes in the hidden layer. 34 35 activation_name: str. 36 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'. 37 38 a: float. 39 hyperparameter for 'prelu' or 'elu' activation function. 40 41 nodes_sim: str. 42 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 43 'uniform'. 44 45 bias: boolean. 46 indicates if the hidden layer contains a bias term (True) or not 47 (False). 48 49 dropout: float. 50 regularization parameter; (random) percentage of nodes dropped out 51 of the training. 52 53 direct_link: boolean. 54 indicates if the original predictors are included (True) in model's fitting or not (False). 55 56 n_clusters: int. 57 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering). 58 59 cluster_encode: bool. 60 defines how the variable containing clusters is treated (default is one-hot) 61 if `False`, then labels are used, without one-hot encoding. 62 63 type_clust: str. 64 type of clustering method: currently k-means ('kmeans') or Gaussian 65 Mixture Model ('gmm'). 66 67 type_scaling: a tuple of 3 strings. 68 scaling methods for inputs, hidden layer, and clustering respectively 69 (and when relevant). 70 Currently available: standardization ('std') or MinMax scaling ('minmax'). 71 72 lags: int. 73 number of lags used for each time series. 74 75 replications: int. 76 number of replications (if needed, for predictive simulation). Default is 'None'. 77 78 kernel: str. 79 the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'. 80 81 agg: str. 82 either "mean" or "median" for simulation of bootstrap aggregating 83 84 seed: int. 85 reproducibility seed for nodes_sim=='uniform' or predictive simulation. 86 87 backend: str. 88 "cpu" or "gpu" or "tpu". 89 90 verbose: int. 91 0: not printing; 1: printing 92 93 show_progress: bool. 94 True: progress bar when fitting each series; False: no progress bar when fitting each series 95 96 Attributes: 97 98 fit_objs_: dict 99 objects adjusted to each individual time series 100 101 y_: {array-like} 102 MTS responses (most recent observations first) 103 104 X_: {array-like} 105 MTS lags 106 107 xreg_: {array-like} 108 external regressors 109 110 y_means_: dict 111 a dictionary of each series mean values 112 113 preds_: {array-like} 114 successive model predictions 115 116 preds_std_: {array-like} 117 standard deviation around the predictions 118 119 return_std_: boolean 120 return uncertainty or not (set in predict) 121 122 df_: data frame 123 the input data frame, in case a data.frame is provided to `fit` 124 125 Examples: 126 127 Example 1: 128 129 ```python 130 import nnetsauce as ns 131 import numpy as np 132 from sklearn import linear_model 133 np.random.seed(123) 134 135 M = np.random.rand(10, 3) 136 M[:,0] = 10*M[:,0] 137 M[:,2] = 25*M[:,2] 138 print(M) 139 140 # Adjust Bayesian Ridge 141 regr4 = linear_model.BayesianRidge() 142 obj_MTS = ns.MTS(regr4, lags = 1, n_hidden_features=5) 143 obj_MTS.fit(M) 144 print(obj_MTS.predict()) 145 146 # with credible intervals 147 print(obj_MTS.predict(return_std=True, level=80)) 148 149 print(obj_MTS.predict(return_std=True, level=95)) 150 ``` 151 152 Example 2: 153 154 ```python 155 import nnetsauce as ns 156 import numpy as np 157 from sklearn import linear_model 158 159 dataset = { 160 'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'], 161 'series1' : [34, 30, 35.6, 33.3, 38.1], 162 'series2' : [4, 5.5, 5.6, 6.3, 5.1], 163 'series3' : [100, 100.5, 100.6, 100.2, 100.1]} 164 df = pd.DataFrame(dataset).set_index('date') 165 print(df) 166 167 # Adjust Bayesian Ridge 168 regr5 = linear_model.BayesianRidge() 169 obj_MTS = ns.MTS(regr5, lags = 1, n_hidden_features=5) 170 obj_MTS.fit(df) 171 print(obj_MTS.predict()) 172 173 # with credible intervals 174 print(obj_MTS.predict(return_std=True, level=80)) 175 176 print(obj_MTS.predict(return_std=True, level=95)) 177 ``` 178 """ 179 180 # construct the object ----- 181 182 def __init__( 183 self, 184 obj, 185 n_hidden_features=5, 186 activation_name="relu", 187 a=0.01, 188 nodes_sim="sobol", 189 bias=True, 190 dropout=0, 191 direct_link=True, 192 n_clusters=2, 193 cluster_encode=True, 194 type_clust="kmeans", 195 type_scaling=("std", "std", "std"), 196 lags=1, 197 type_pi="kde", 198 replications=None, 199 kernel=None, 200 agg="mean", 201 seed=123, 202 backend="cpu", 203 verbose=0, 204 show_progress=True, 205 ): 206 assert int(lags) == lags, "parameter 'lags' should be an integer" 207 208 super().__init__( 209 n_hidden_features=n_hidden_features, 210 activation_name=activation_name, 211 a=a, 212 nodes_sim=nodes_sim, 213 bias=bias, 214 dropout=dropout, 215 direct_link=direct_link, 216 n_clusters=n_clusters, 217 cluster_encode=cluster_encode, 218 type_clust=type_clust, 219 type_scaling=type_scaling, 220 seed=seed, 221 backend=backend, 222 ) 223 224 self.obj = obj 225 self.n_series = None 226 self.lags = lags 227 self.type_pi = type_pi 228 self.replications = replications 229 self.kernel = kernel 230 self.agg = agg 231 self.verbose = verbose 232 self.show_progress = show_progress 233 self.series_names = None 234 self.input_dates = None 235 self.fit_objs_ = {} 236 self.y_ = None # MTS responses (most recent observations first) 237 self.X_ = None # MTS lags 238 self.xreg_ = None 239 self.y_means_ = {} 240 self.mean_ = None 241 self.upper_ = None 242 self.lower_ = None 243 self.output_dates_ = None 244 self.preds_std_ = [] 245 self.alpha_ = None 246 self.return_std_ = None 247 self.df_ = None 248 self.residuals_ = [] 249 self.residuals_sims_ = None 250 self.kde_ = None 251 self.sims_ = None 252 253 def fit(self, X, xreg=None, **kwargs): 254 """Fit MTS model to training data X, with optional regressors xreg 255 256 Parameters: 257 258 X: {array-like}, shape = [n_samples, n_features] 259 Training time series, where n_samples is the number 260 of samples and n_features is the number of features; 261 X must be in increasing order (most recent observations last) 262 263 xreg: {array-like}, shape = [n_samples, n_features_xreg] 264 Additional regressors to be passed to obj 265 xreg must be in increasing order (most recent observations last) 266 267 **kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity) 268 269 Returns: 270 271 self: object 272 """ 273 274 if isinstance(X, pd.DataFrame) is False: 275 self.series_names = ["series" + str(i) for i in range(X.shape[1])] 276 X = pd.DataFrame(X, columns=self.series_names) 277 else: 278 X = copy.deepcopy(mo.convert_df_to_numeric(X)) 279 self.series_names = list(X.columns.values) 280 281 self.df_ = X 282 X = X.values 283 self.input_dates = ts.compute_input_dates(self.df_) 284 285 try: 286 # multivariate time series 287 n, p = X.shape 288 except: 289 # univariate time series 290 n = X.shape[0] 291 p = 1 292 293 rep_1_n = np.repeat(1, n) 294 295 self.y_ = None 296 self.X_ = None 297 self.n_series = p 298 self.fit_objs_.clear() 299 self.y_means_.clear() 300 residuals_ = [] 301 self.residuals_ = None 302 self.residuals_sims_ = None 303 self.kde_ = None 304 self.sims_ = None 305 self.scaled_Z_ = None 306 self.centered_y_is_ = [] 307 308 if p > 1: 309 # multivariate time series 310 mts_input = ts.create_train_inputs(X[::-1], self.lags) 311 else: 312 # univariate time series 313 mts_input = ts.create_train_inputs( 314 X.reshape(-1, 1)[::-1], self.lags 315 ) 316 317 self.y_ = mts_input[0] 318 319 self.X_ = mts_input[1] 320 321 if xreg is not None: 322 if isinstance(xreg, pd.DataFrame): 323 xreg = xreg.values 324 325 assert ( 326 xreg.shape[0] == n 327 ), "'xreg' and 'X' must have the same number of observations" 328 329 self.xreg_ = xreg 330 331 xreg_input = ts.create_train_inputs(xreg[::-1], self.lags) 332 333 dummy_y, scaled_Z = self.cook_training_set( 334 y=rep_1_n, 335 X=mo.cbind(self.X_, xreg_input[1], backend=self.backend), 336 ) 337 338 else: # xreg is None 339 # avoids scaling X p times in the loop 340 dummy_y, scaled_Z = self.cook_training_set(y=rep_1_n, X=self.X_) 341 342 self.scaled_Z_ = scaled_Z 343 344 # loop on all the time series and adjust self.obj.fit 345 if self.verbose > 0: 346 print( 347 f"\n Adjusting {type(self.obj).__name__} to multivariate time series... \n " 348 ) 349 350 if self.show_progress is True: 351 iterator = tqdm(range(p)) 352 else: 353 iterator = range(p) 354 355 for i in iterator: 356 y_mean = np.mean(self.y_[:, i]) 357 self.y_means_[i] = y_mean 358 centered_y_i = self.y_[:, i] - y_mean 359 self.centered_y_is_.append(centered_y_i) 360 self.obj.fit(X=scaled_Z, y=centered_y_i) 361 self.fit_objs_[i] = deepcopy(self.obj) 362 residuals_.append( 363 (centered_y_i - self.fit_objs_[i].predict(scaled_Z)).tolist() 364 ) 365 366 self.residuals_ = np.asarray(residuals_).T 367 368 if self.replications != None and self.type_pi == "kde": 369 if self.verbose > 0: 370 print(f"\n Simulate residuals using {self.kernel} kernel... \n") 371 assert self.kernel in ( 372 "gaussian", 373 "tophat", 374 ), "currently, 'kernel' must be either 'gaussian' or 'tophat'" 375 kernel_bandwidths = {"bandwidth": np.logspace(-6, 6, 150)} 376 grid = GridSearchCV( 377 KernelDensity(kernel=self.kernel, **kwargs), 378 param_grid=kernel_bandwidths, 379 ) 380 grid.fit(self.residuals_) 381 382 if self.verbose > 0: 383 print( 384 f"\n Best parameters for {self.kernel} kernel: {grid.best_params_} \n" 385 ) 386 387 self.kde_ = grid.best_estimator_ 388 389 return self 390 391 def predict(self, h=5, level=95, new_xreg=None, **kwargs): 392 """Forecast all the time series, h steps ahead 393 394 Parameters: 395 396 h: {integer} 397 Forecasting horizon 398 399 level: {integer} 400 Level of confidence (if obj has option 'return_std' and the 401 posterior is gaussian) 402 403 new_xreg: {array-like}, shape = [n_samples = h, n_new_xreg] 404 New values of additional (deterministic) regressors on horizon = h 405 new_xreg must be in increasing order (most recent observations last) 406 407 **kwargs: additional parameters to be passed to 408 self.cook_test_set 409 410 Returns: 411 412 model predictions for horizon = h: {array-like}, data frame or tuple. 413 Standard deviation and prediction intervals are returned when 414 `obj.predict` can return standard deviation 415 """ 416 417 self.output_dates_, frequency = ts.compute_output_dates(self.df_, h) 418 419 self.return_std_ = False # do not remove (/!\) 420 421 self.mean_ = None # do not remove (/!\) 422 423 self.mean_ = deepcopy(self.y_) # do not remove (/!\) 424 425 self.lower_ = None 426 427 self.upper_ = None 428 429 self.sims_ = None 430 431 y_means_ = np.asarray([self.y_means_[i] for i in range(self.n_series)]) 432 433 n_features = self.n_series * self.lags 434 435 self.alpha_ = 100 - level 436 437 if "return_std" in kwargs: 438 self.return_std_ = True 439 self.preds_std_ = [] 440 pi_multiplier = norm.ppf(1 - self.alpha_ / 200) 441 DescribeResult = namedtuple( 442 "DescribeResult", ("mean", "lower", "upper") 443 ) # to be updated 444 445 if self.xreg_ is None: # no external regressors 446 if self.kde_ != None and self.type_pi == "kde": 447 self.residuals_sims_ = tuple( 448 self.kde_.sample( 449 n_samples=h, random_state=self.seed + 100 * i 450 ) 451 for i in tqdm(range(self.replications)) 452 ) 453 454 for _ in range(h): 455 new_obs = ts.reformat_response(self.mean_, self.lags) 456 457 new_X = new_obs.reshape(1, n_features) 458 459 cooked_new_X = self.cook_test_set(new_X, **kwargs) 460 461 if "return_std" in kwargs: 462 self.preds_std_.append( 463 [ 464 np.asarray( 465 self.fit_objs_[i].predict( 466 cooked_new_X, return_std=True 467 )[1] 468 ).item() 469 for i in range(self.n_series) 470 ] 471 ) 472 473 predicted_cooked_new_X = np.asarray( 474 [ 475 np.asarray( 476 self.fit_objs_[i].predict(cooked_new_X) 477 ).item() 478 for i in range(self.n_series) 479 ] 480 ) 481 482 preds = np.asarray(y_means_ + predicted_cooked_new_X) 483 484 self.mean_ = mo.rbind(preds, self.mean_) 485 486 else: # if self.xreg_ is not None: # with external regressors 487 assert ( 488 new_xreg is not None 489 ), "'new_xreg' must be provided to predict()" 490 491 if isinstance(new_xreg, pd.DataFrame): 492 new_xreg = new_xreg.values 493 494 if self.kde_ is not None: 495 if self.verbose > 0: 496 print("\n Obtain simulations for adjusted residuals... \n") 497 self.residuals_sims_ = tuple( 498 self.kde_.sample( 499 n_samples=h, random_state=self.seed + 100 * i 500 ) 501 for i in tqdm(range(self.replications)) 502 ) 503 504 try: 505 n_obs_xreg, n_features_xreg = new_xreg.shape 506 assert ( 507 n_features_xreg == self.xreg_.shape[1] 508 ), "check number of inputs provided for 'new_xreg' (compare with self.xreg_.shape[1])" 509 except: 510 n_obs_xreg = new_xreg.shape # one series 511 512 assert ( 513 n_obs_xreg == h 514 ), "please provide values of regressors 'new_xreg' for the whole horizon 'h'" 515 516 n_features_xreg = n_features_xreg * self.lags 517 518 inv_new_xreg = mo.rbind(self.xreg_, new_xreg)[::-1] 519 520 for _ in range(h): 521 new_obs = ts.reformat_response(self.mean_, self.lags) 522 523 new_obs_xreg = ts.reformat_response(inv_new_xreg, self.lags) 524 525 new_X = np.concatenate( 526 (new_obs, new_obs_xreg), axis=None 527 ).reshape(1, -1) 528 529 cooked_new_X = self.cook_test_set(new_X, **kwargs) 530 531 if "return_std" in kwargs: 532 self.preds_std_.append( 533 [ 534 np.asarray( 535 self.fit_objs_[i].predict( 536 cooked_new_X, return_std=True 537 )[1] 538 ).item() 539 for i in range(self.n_series) 540 ] 541 ) 542 543 predicted_cooked_new_X = np.asarray( 544 [ 545 np.asarray( 546 self.fit_objs_[i].predict(cooked_new_X) 547 ).item() 548 for i in range(self.n_series) 549 ] 550 ) 551 552 preds = np.asarray(y_means_ + predicted_cooked_new_X) 553 554 self.mean_ = mo.rbind(preds, self.mean_) 555 556 # function's return ---------------------------------------------------------------------- 557 self.mean_ = pd.DataFrame( 558 self.mean_[0:h, :][::-1], 559 columns=self.df_.columns, 560 index=self.output_dates_, 561 ) 562 if "return_std" not in kwargs: 563 if self.kde_ is None: 564 return self.mean_ 565 566 # if "return_std" not in kwargs and self.kde_ is not None 567 meanf = [] 568 lower = [] 569 upper = [] 570 self.sims_ = tuple( 571 ( 572 self.mean_ + self.residuals_sims_[i] 573 for i in tqdm(range(self.replications)) 574 ) 575 ) 576 DescribeResult = namedtuple( 577 "DescribeResult", ("mean", "sims", "lower", "upper") 578 ) 579 for ix in range(self.n_series): 580 sims_ix = getsims(self.sims_, ix) 581 if self.agg == "mean": 582 meanf.append(np.mean(sims_ix, axis=1)) 583 else: 584 meanf.append(np.median(sims_ix, axis=1)) 585 lower.append(np.quantile(sims_ix, q=self.alpha_ / 200, axis=1)) 586 upper.append( 587 np.quantile(sims_ix, q=1 - self.alpha_ / 200, axis=1) 588 ) 589 590 self.mean_ = pd.DataFrame( 591 np.asarray(meanf).T, 592 columns=self.df_.columns, 593 index=self.output_dates_, 594 ) 595 596 self.lower_ = pd.DataFrame( 597 np.asarray(lower).T, 598 columns=self.df_.columns, 599 index=self.output_dates_, 600 ) 601 602 self.upper_ = pd.DataFrame( 603 np.asarray(upper).T, 604 columns=self.df_.columns, 605 index=self.output_dates_, 606 ) 607 608 return DescribeResult( 609 self.mean_, self.sims_, self.lower_, self.upper_ 610 ) 611 612 # if "return_std" in kwargs 613 DescribeResult = namedtuple( 614 "DescribeResult", ("mean", "lower", "upper") 615 ) 616 617 self.mean_ = pd.DataFrame( 618 np.asarray(self.mean_), 619 columns=self.df_.columns, 620 index=self.output_dates_, 621 ) 622 623 self.preds_std_ = np.asarray(self.preds_std_) 624 625 self.lower_ = pd.DataFrame( 626 self.mean_.values - pi_multiplier * self.preds_std_, 627 columns=self.df_.columns, 628 index=self.output_dates_, 629 ) 630 631 self.upper_ = pd.DataFrame( 632 self.mean_.values + pi_multiplier * self.preds_std_, 633 columns=self.df_.columns, 634 index=self.output_dates_, 635 ) 636 637 return DescribeResult(self.mean_, self.lower_, self.upper_) 638 639 def score(self, X, training_index, testing_index, scoring=None, **kwargs): 640 """Train on training_index, score on testing_index.""" 641 642 assert ( 643 bool(set(training_index).intersection(set(testing_index))) == False 644 ), "Non-overlapping 'training_index' and 'testing_index' required" 645 646 # Dimensions 647 try: 648 # multivariate time series 649 n, p = X.shape 650 except: 651 # univariate time series 652 n = X.shape[0] 653 p = 1 654 655 # Training and testing sets 656 if p > 1: 657 X_train = X[training_index, :] 658 X_test = X[testing_index, :] 659 else: 660 X_train = X[training_index] 661 X_test = X[testing_index] 662 663 # Horizon 664 h = len(testing_index) 665 assert ( 666 len(training_index) + h 667 ) <= n, "Please check lengths of training and testing windows" 668 669 # Fit and predict 670 self.fit(X_train, **kwargs) 671 preds = self.predict(h=h, **kwargs) 672 673 if scoring is None: 674 scoring = "neg_root_mean_squared_error" 675 676 # check inputs 677 assert scoring in ( 678 "explained_variance", 679 "neg_mean_absolute_error", 680 "neg_mean_squared_error", 681 "neg_root_mean_squared_error", 682 "neg_mean_squared_log_error", 683 "neg_median_absolute_error", 684 "r2", 685 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 686 'neg_mean_squared_error', 'neg_root_mean_squared_error', 'neg_mean_squared_log_error', \ 687 'neg_median_absolute_error', 'r2')" 688 689 scoring_options = { 690 "explained_variance": skm2.explained_variance_score, 691 "neg_mean_absolute_error": skm2.mean_absolute_error, 692 "neg_mean_squared_error": skm2.mean_squared_error, 693 "neg_root_mean_squared_error": lambda x, y: np.sqrt( 694 skm2.mean_squared_error(x, y) 695 ), 696 "neg_mean_squared_log_error": skm2.mean_squared_log_error, 697 "neg_median_absolute_error": skm2.median_absolute_error, 698 "r2": skm2.r2_score, 699 } 700 701 # if p > 1: 702 # return tuple( 703 # [ 704 # scoring_options[scoring]( 705 # X_test[:, i], preds[:, i]#, **kwargs 706 # ) 707 # for i in range(p) 708 # ] 709 # ) 710 # else: 711 return scoring_options[scoring](X_test, preds) 712 713 def plot(self, series, type_axis="dates", type_plot="pi"): 714 """Plot time series forecast 715 716 Parameters: 717 718 series: {integer} or {string} 719 series index or name 720 """ 721 722 assert all( 723 [ 724 self.mean_ is not None, 725 self.lower_ is not None, 726 self.upper_ is not None, 727 self.output_dates_ is not None, 728 ] 729 ), "model forecasting must be obtained first (with predict)" 730 731 if isinstance(series, str): 732 assert ( 733 series in self.series_names 734 ), f"series {series} doesn't exist in the input dataset" 735 series_idx = self.df_.columns.get_loc(series) 736 else: 737 assert isinstance(series, int) and ( 738 0 <= series < self.n_series 739 ), f"check series index (< {self.n_series})" 740 series_idx = series 741 742 y_all = list(self.df_.iloc[:, series_idx]) + list( 743 self.mean_.iloc[:, series_idx] 744 ) 745 y_test = list(self.mean_.iloc[:, series_idx]) 746 n_points_all = len(y_all) 747 n_points_train = self.df_.shape[0] 748 749 if type_axis == "numeric": 750 x_all = [i for i in range(n_points_all)] 751 x_test = [i for i in range(n_points_train, n_points_all)] 752 753 if type_axis == "dates": # use dates 754 x_all = np.concatenate( 755 (self.input_dates.values, self.output_dates_.values), axis=None 756 ) 757 x_test = self.output_dates_.values 758 759 if type_plot == "pi": 760 fig, ax = plt.subplots() 761 ax.plot(x_all, y_all, "-") 762 ax.plot(x_test, y_test, "-", color="orange") 763 ax.fill_between( 764 x_test, 765 self.lower_.iloc[:, series_idx], 766 self.upper_.iloc[:, series_idx], 767 alpha=0.2, 768 color="orange", 769 ) 770 plt.title( 771 f"prediction intervals for {self.replications} simulations of {series}", 772 loc="left", 773 fontsize=12, 774 fontweight=0, 775 color="black", 776 ) 777 plt.show() 778 779 if type_plot == "spaghetti": 780 palette = plt.get_cmap("Set1") 781 sims_ix = getsims(self.sims_, series_idx) 782 plt.plot(x_all, y_all, "-") 783 for col_ix in range( 784 sims_ix.shape[1] 785 ): # avoid this when there are thousands of simulations 786 plt.plot( 787 x_test, 788 sims_ix[:, col_ix], 789 "-", 790 color=palette(col_ix), 791 linewidth=1, 792 alpha=0.9, 793 ) 794 plt.plot(x_all, y_all, "-", color="black") 795 plt.plot(x_test, y_test, "-", color="blue") 796 # Add titles 797 plt.title( 798 f"{self.replications} simulations of {series}", 799 loc="left", 800 fontsize=12, 801 fontweight=0, 802 color="black", 803 ) 804 plt.xlabel("Time") 805 plt.ylabel("Values") 806 # Show the graph 807 plt.show()
Univariate and multivariate time series (MTS) forecasting with Quasi-Randomized networks (Work in progress /!)
Parameters:
obj: object.
any object containing a method fit (obj.fit()) and a method predict
(obj.predict()).
n_hidden_features: int.
number of nodes in the hidden layer.
activation_name: str.
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'.
a: float.
hyperparameter for 'prelu' or 'elu' activation function.
nodes_sim: str.
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'.
bias: boolean.
indicates if the hidden layer contains a bias term (True) or not
(False).
dropout: float.
regularization parameter; (random) percentage of nodes dropped out
of the training.
direct_link: boolean.
indicates if the original predictors are included (True) in model's fitting or not (False).
n_clusters: int.
number of clusters for 'kmeans' or 'gmm' clustering (could be 0: no clustering).
cluster_encode: bool.
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding.
type_clust: str.
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm').
type_scaling: a tuple of 3 strings.
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax').
lags: int.
number of lags used for each time series.
replications: int.
number of replications (if needed, for predictive simulation). Default is 'None'.
kernel: str.
the kernel to use for residuals density estimation (used for predictive simulation). Currently, either 'gaussian' or 'tophat'.
agg: str.
either "mean" or "median" for simulation of bootstrap aggregating
seed: int.
reproducibility seed for nodes_sim=='uniform' or predictive simulation.
backend: str.
"cpu" or "gpu" or "tpu".
verbose: int.
0: not printing; 1: printing
show_progress: bool.
True: progress bar when fitting each series; False: no progress bar when fitting each series
Attributes:
fit_objs_: dict
objects adjusted to each individual time series
y_: {array-like}
MTS responses (most recent observations first)
X_: {array-like}
MTS lags
xreg_: {array-like}
external regressors
y_means_: dict
a dictionary of each series mean values
preds_: {array-like}
successive model predictions
preds_std_: {array-like}
standard deviation around the predictions
return_std_: boolean
return uncertainty or not (set in predict)
df_: data frame
the input data frame, in case a data.frame is provided to `fit`
Examples:
Example 1:
import nnetsauce as ns
import numpy as np
from sklearn import linear_model
np.random.seed(123)
M = np.random.rand(10, 3)
M[:,0] = 10*M[:,0]
M[:,2] = 25*M[:,2]
print(M)
# Adjust Bayesian Ridge
regr4 = linear_model.BayesianRidge()
obj_MTS = ns.MTS(regr4, lags = 1, n_hidden_features=5)
obj_MTS.fit(M)
print(obj_MTS.predict())
# with credible intervals
print(obj_MTS.predict(return_std=True, level=80))
print(obj_MTS.predict(return_std=True, level=95))
Example 2:
import nnetsauce as ns
import numpy as np
from sklearn import linear_model
dataset = {
'date' : ['2001-01-01', '2002-01-01', '2003-01-01', '2004-01-01', '2005-01-01'],
'series1' : [34, 30, 35.6, 33.3, 38.1],
'series2' : [4, 5.5, 5.6, 6.3, 5.1],
'series3' : [100, 100.5, 100.6, 100.2, 100.1]}
df = pd.DataFrame(dataset).set_index('date')
print(df)
# Adjust Bayesian Ridge
regr5 = linear_model.BayesianRidge()
obj_MTS = ns.MTS(regr5, lags = 1, n_hidden_features=5)
obj_MTS.fit(df)
print(obj_MTS.predict())
# with credible intervals
print(obj_MTS.predict(return_std=True, level=80))
print(obj_MTS.predict(return_std=True, level=95))
182 def __init__( 183 self, 184 obj, 185 n_hidden_features=5, 186 activation_name="relu", 187 a=0.01, 188 nodes_sim="sobol", 189 bias=True, 190 dropout=0, 191 direct_link=True, 192 n_clusters=2, 193 cluster_encode=True, 194 type_clust="kmeans", 195 type_scaling=("std", "std", "std"), 196 lags=1, 197 type_pi="kde", 198 replications=None, 199 kernel=None, 200 agg="mean", 201 seed=123, 202 backend="cpu", 203 verbose=0, 204 show_progress=True, 205 ): 206 assert int(lags) == lags, "parameter 'lags' should be an integer" 207 208 super().__init__( 209 n_hidden_features=n_hidden_features, 210 activation_name=activation_name, 211 a=a, 212 nodes_sim=nodes_sim, 213 bias=bias, 214 dropout=dropout, 215 direct_link=direct_link, 216 n_clusters=n_clusters, 217 cluster_encode=cluster_encode, 218 type_clust=type_clust, 219 type_scaling=type_scaling, 220 seed=seed, 221 backend=backend, 222 ) 223 224 self.obj = obj 225 self.n_series = None 226 self.lags = lags 227 self.type_pi = type_pi 228 self.replications = replications 229 self.kernel = kernel 230 self.agg = agg 231 self.verbose = verbose 232 self.show_progress = show_progress 233 self.series_names = None 234 self.input_dates = None 235 self.fit_objs_ = {} 236 self.y_ = None # MTS responses (most recent observations first) 237 self.X_ = None # MTS lags 238 self.xreg_ = None 239 self.y_means_ = {} 240 self.mean_ = None 241 self.upper_ = None 242 self.lower_ = None 243 self.output_dates_ = None 244 self.preds_std_ = [] 245 self.alpha_ = None 246 self.return_std_ = None 247 self.df_ = None 248 self.residuals_ = [] 249 self.residuals_sims_ = None 250 self.kde_ = None 251 self.sims_ = None
253 def fit(self, X, xreg=None, **kwargs): 254 """Fit MTS model to training data X, with optional regressors xreg 255 256 Parameters: 257 258 X: {array-like}, shape = [n_samples, n_features] 259 Training time series, where n_samples is the number 260 of samples and n_features is the number of features; 261 X must be in increasing order (most recent observations last) 262 263 xreg: {array-like}, shape = [n_samples, n_features_xreg] 264 Additional regressors to be passed to obj 265 xreg must be in increasing order (most recent observations last) 266 267 **kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity) 268 269 Returns: 270 271 self: object 272 """ 273 274 if isinstance(X, pd.DataFrame) is False: 275 self.series_names = ["series" + str(i) for i in range(X.shape[1])] 276 X = pd.DataFrame(X, columns=self.series_names) 277 else: 278 X = copy.deepcopy(mo.convert_df_to_numeric(X)) 279 self.series_names = list(X.columns.values) 280 281 self.df_ = X 282 X = X.values 283 self.input_dates = ts.compute_input_dates(self.df_) 284 285 try: 286 # multivariate time series 287 n, p = X.shape 288 except: 289 # univariate time series 290 n = X.shape[0] 291 p = 1 292 293 rep_1_n = np.repeat(1, n) 294 295 self.y_ = None 296 self.X_ = None 297 self.n_series = p 298 self.fit_objs_.clear() 299 self.y_means_.clear() 300 residuals_ = [] 301 self.residuals_ = None 302 self.residuals_sims_ = None 303 self.kde_ = None 304 self.sims_ = None 305 self.scaled_Z_ = None 306 self.centered_y_is_ = [] 307 308 if p > 1: 309 # multivariate time series 310 mts_input = ts.create_train_inputs(X[::-1], self.lags) 311 else: 312 # univariate time series 313 mts_input = ts.create_train_inputs( 314 X.reshape(-1, 1)[::-1], self.lags 315 ) 316 317 self.y_ = mts_input[0] 318 319 self.X_ = mts_input[1] 320 321 if xreg is not None: 322 if isinstance(xreg, pd.DataFrame): 323 xreg = xreg.values 324 325 assert ( 326 xreg.shape[0] == n 327 ), "'xreg' and 'X' must have the same number of observations" 328 329 self.xreg_ = xreg 330 331 xreg_input = ts.create_train_inputs(xreg[::-1], self.lags) 332 333 dummy_y, scaled_Z = self.cook_training_set( 334 y=rep_1_n, 335 X=mo.cbind(self.X_, xreg_input[1], backend=self.backend), 336 ) 337 338 else: # xreg is None 339 # avoids scaling X p times in the loop 340 dummy_y, scaled_Z = self.cook_training_set(y=rep_1_n, X=self.X_) 341 342 self.scaled_Z_ = scaled_Z 343 344 # loop on all the time series and adjust self.obj.fit 345 if self.verbose > 0: 346 print( 347 f"\n Adjusting {type(self.obj).__name__} to multivariate time series... \n " 348 ) 349 350 if self.show_progress is True: 351 iterator = tqdm(range(p)) 352 else: 353 iterator = range(p) 354 355 for i in iterator: 356 y_mean = np.mean(self.y_[:, i]) 357 self.y_means_[i] = y_mean 358 centered_y_i = self.y_[:, i] - y_mean 359 self.centered_y_is_.append(centered_y_i) 360 self.obj.fit(X=scaled_Z, y=centered_y_i) 361 self.fit_objs_[i] = deepcopy(self.obj) 362 residuals_.append( 363 (centered_y_i - self.fit_objs_[i].predict(scaled_Z)).tolist() 364 ) 365 366 self.residuals_ = np.asarray(residuals_).T 367 368 if self.replications != None and self.type_pi == "kde": 369 if self.verbose > 0: 370 print(f"\n Simulate residuals using {self.kernel} kernel... \n") 371 assert self.kernel in ( 372 "gaussian", 373 "tophat", 374 ), "currently, 'kernel' must be either 'gaussian' or 'tophat'" 375 kernel_bandwidths = {"bandwidth": np.logspace(-6, 6, 150)} 376 grid = GridSearchCV( 377 KernelDensity(kernel=self.kernel, **kwargs), 378 param_grid=kernel_bandwidths, 379 ) 380 grid.fit(self.residuals_) 381 382 if self.verbose > 0: 383 print( 384 f"\n Best parameters for {self.kernel} kernel: {grid.best_params_} \n" 385 ) 386 387 self.kde_ = grid.best_estimator_ 388 389 return self
Fit MTS model to training data X, with optional regressors xreg
Parameters:
X: {array-like}, shape = [n_samples, n_features]
Training time series, where n_samples is the number
of samples and n_features is the number of features;
X must be in increasing order (most recent observations last)
xreg: {array-like}, shape = [n_samples, n_features_xreg]
Additional regressors to be passed to obj
xreg must be in increasing order (most recent observations last)
**kwargs: for now, additional parameters to be passed to for kernel density estimation, when needed (see sklearn.neighbors.KernelDensity)
Returns:
self: object
391 def predict(self, h=5, level=95, new_xreg=None, **kwargs): 392 """Forecast all the time series, h steps ahead 393 394 Parameters: 395 396 h: {integer} 397 Forecasting horizon 398 399 level: {integer} 400 Level of confidence (if obj has option 'return_std' and the 401 posterior is gaussian) 402 403 new_xreg: {array-like}, shape = [n_samples = h, n_new_xreg] 404 New values of additional (deterministic) regressors on horizon = h 405 new_xreg must be in increasing order (most recent observations last) 406 407 **kwargs: additional parameters to be passed to 408 self.cook_test_set 409 410 Returns: 411 412 model predictions for horizon = h: {array-like}, data frame or tuple. 413 Standard deviation and prediction intervals are returned when 414 `obj.predict` can return standard deviation 415 """ 416 417 self.output_dates_, frequency = ts.compute_output_dates(self.df_, h) 418 419 self.return_std_ = False # do not remove (/!\) 420 421 self.mean_ = None # do not remove (/!\) 422 423 self.mean_ = deepcopy(self.y_) # do not remove (/!\) 424 425 self.lower_ = None 426 427 self.upper_ = None 428 429 self.sims_ = None 430 431 y_means_ = np.asarray([self.y_means_[i] for i in range(self.n_series)]) 432 433 n_features = self.n_series * self.lags 434 435 self.alpha_ = 100 - level 436 437 if "return_std" in kwargs: 438 self.return_std_ = True 439 self.preds_std_ = [] 440 pi_multiplier = norm.ppf(1 - self.alpha_ / 200) 441 DescribeResult = namedtuple( 442 "DescribeResult", ("mean", "lower", "upper") 443 ) # to be updated 444 445 if self.xreg_ is None: # no external regressors 446 if self.kde_ != None and self.type_pi == "kde": 447 self.residuals_sims_ = tuple( 448 self.kde_.sample( 449 n_samples=h, random_state=self.seed + 100 * i 450 ) 451 for i in tqdm(range(self.replications)) 452 ) 453 454 for _ in range(h): 455 new_obs = ts.reformat_response(self.mean_, self.lags) 456 457 new_X = new_obs.reshape(1, n_features) 458 459 cooked_new_X = self.cook_test_set(new_X, **kwargs) 460 461 if "return_std" in kwargs: 462 self.preds_std_.append( 463 [ 464 np.asarray( 465 self.fit_objs_[i].predict( 466 cooked_new_X, return_std=True 467 )[1] 468 ).item() 469 for i in range(self.n_series) 470 ] 471 ) 472 473 predicted_cooked_new_X = np.asarray( 474 [ 475 np.asarray( 476 self.fit_objs_[i].predict(cooked_new_X) 477 ).item() 478 for i in range(self.n_series) 479 ] 480 ) 481 482 preds = np.asarray(y_means_ + predicted_cooked_new_X) 483 484 self.mean_ = mo.rbind(preds, self.mean_) 485 486 else: # if self.xreg_ is not None: # with external regressors 487 assert ( 488 new_xreg is not None 489 ), "'new_xreg' must be provided to predict()" 490 491 if isinstance(new_xreg, pd.DataFrame): 492 new_xreg = new_xreg.values 493 494 if self.kde_ is not None: 495 if self.verbose > 0: 496 print("\n Obtain simulations for adjusted residuals... \n") 497 self.residuals_sims_ = tuple( 498 self.kde_.sample( 499 n_samples=h, random_state=self.seed + 100 * i 500 ) 501 for i in tqdm(range(self.replications)) 502 ) 503 504 try: 505 n_obs_xreg, n_features_xreg = new_xreg.shape 506 assert ( 507 n_features_xreg == self.xreg_.shape[1] 508 ), "check number of inputs provided for 'new_xreg' (compare with self.xreg_.shape[1])" 509 except: 510 n_obs_xreg = new_xreg.shape # one series 511 512 assert ( 513 n_obs_xreg == h 514 ), "please provide values of regressors 'new_xreg' for the whole horizon 'h'" 515 516 n_features_xreg = n_features_xreg * self.lags 517 518 inv_new_xreg = mo.rbind(self.xreg_, new_xreg)[::-1] 519 520 for _ in range(h): 521 new_obs = ts.reformat_response(self.mean_, self.lags) 522 523 new_obs_xreg = ts.reformat_response(inv_new_xreg, self.lags) 524 525 new_X = np.concatenate( 526 (new_obs, new_obs_xreg), axis=None 527 ).reshape(1, -1) 528 529 cooked_new_X = self.cook_test_set(new_X, **kwargs) 530 531 if "return_std" in kwargs: 532 self.preds_std_.append( 533 [ 534 np.asarray( 535 self.fit_objs_[i].predict( 536 cooked_new_X, return_std=True 537 )[1] 538 ).item() 539 for i in range(self.n_series) 540 ] 541 ) 542 543 predicted_cooked_new_X = np.asarray( 544 [ 545 np.asarray( 546 self.fit_objs_[i].predict(cooked_new_X) 547 ).item() 548 for i in range(self.n_series) 549 ] 550 ) 551 552 preds = np.asarray(y_means_ + predicted_cooked_new_X) 553 554 self.mean_ = mo.rbind(preds, self.mean_) 555 556 # function's return ---------------------------------------------------------------------- 557 self.mean_ = pd.DataFrame( 558 self.mean_[0:h, :][::-1], 559 columns=self.df_.columns, 560 index=self.output_dates_, 561 ) 562 if "return_std" not in kwargs: 563 if self.kde_ is None: 564 return self.mean_ 565 566 # if "return_std" not in kwargs and self.kde_ is not None 567 meanf = [] 568 lower = [] 569 upper = [] 570 self.sims_ = tuple( 571 ( 572 self.mean_ + self.residuals_sims_[i] 573 for i in tqdm(range(self.replications)) 574 ) 575 ) 576 DescribeResult = namedtuple( 577 "DescribeResult", ("mean", "sims", "lower", "upper") 578 ) 579 for ix in range(self.n_series): 580 sims_ix = getsims(self.sims_, ix) 581 if self.agg == "mean": 582 meanf.append(np.mean(sims_ix, axis=1)) 583 else: 584 meanf.append(np.median(sims_ix, axis=1)) 585 lower.append(np.quantile(sims_ix, q=self.alpha_ / 200, axis=1)) 586 upper.append( 587 np.quantile(sims_ix, q=1 - self.alpha_ / 200, axis=1) 588 ) 589 590 self.mean_ = pd.DataFrame( 591 np.asarray(meanf).T, 592 columns=self.df_.columns, 593 index=self.output_dates_, 594 ) 595 596 self.lower_ = pd.DataFrame( 597 np.asarray(lower).T, 598 columns=self.df_.columns, 599 index=self.output_dates_, 600 ) 601 602 self.upper_ = pd.DataFrame( 603 np.asarray(upper).T, 604 columns=self.df_.columns, 605 index=self.output_dates_, 606 ) 607 608 return DescribeResult( 609 self.mean_, self.sims_, self.lower_, self.upper_ 610 ) 611 612 # if "return_std" in kwargs 613 DescribeResult = namedtuple( 614 "DescribeResult", ("mean", "lower", "upper") 615 ) 616 617 self.mean_ = pd.DataFrame( 618 np.asarray(self.mean_), 619 columns=self.df_.columns, 620 index=self.output_dates_, 621 ) 622 623 self.preds_std_ = np.asarray(self.preds_std_) 624 625 self.lower_ = pd.DataFrame( 626 self.mean_.values - pi_multiplier * self.preds_std_, 627 columns=self.df_.columns, 628 index=self.output_dates_, 629 ) 630 631 self.upper_ = pd.DataFrame( 632 self.mean_.values + pi_multiplier * self.preds_std_, 633 columns=self.df_.columns, 634 index=self.output_dates_, 635 ) 636 637 return DescribeResult(self.mean_, self.lower_, self.upper_)
Forecast all the time series, h steps ahead
Parameters:
h: {integer}
Forecasting horizon
level: {integer}
Level of confidence (if obj has option 'return_std' and the
posterior is gaussian)
new_xreg: {array-like}, shape = [n_samples = h, n_new_xreg]
New values of additional (deterministic) regressors on horizon = h
new_xreg must be in increasing order (most recent observations last)
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions for horizon = h: {array-like}, data frame or tuple.
Standard deviation and prediction intervals are returned when
`obj.predict` can return standard deviation
639 def score(self, X, training_index, testing_index, scoring=None, **kwargs): 640 """Train on training_index, score on testing_index.""" 641 642 assert ( 643 bool(set(training_index).intersection(set(testing_index))) == False 644 ), "Non-overlapping 'training_index' and 'testing_index' required" 645 646 # Dimensions 647 try: 648 # multivariate time series 649 n, p = X.shape 650 except: 651 # univariate time series 652 n = X.shape[0] 653 p = 1 654 655 # Training and testing sets 656 if p > 1: 657 X_train = X[training_index, :] 658 X_test = X[testing_index, :] 659 else: 660 X_train = X[training_index] 661 X_test = X[testing_index] 662 663 # Horizon 664 h = len(testing_index) 665 assert ( 666 len(training_index) + h 667 ) <= n, "Please check lengths of training and testing windows" 668 669 # Fit and predict 670 self.fit(X_train, **kwargs) 671 preds = self.predict(h=h, **kwargs) 672 673 if scoring is None: 674 scoring = "neg_root_mean_squared_error" 675 676 # check inputs 677 assert scoring in ( 678 "explained_variance", 679 "neg_mean_absolute_error", 680 "neg_mean_squared_error", 681 "neg_root_mean_squared_error", 682 "neg_mean_squared_log_error", 683 "neg_median_absolute_error", 684 "r2", 685 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 686 'neg_mean_squared_error', 'neg_root_mean_squared_error', 'neg_mean_squared_log_error', \ 687 'neg_median_absolute_error', 'r2')" 688 689 scoring_options = { 690 "explained_variance": skm2.explained_variance_score, 691 "neg_mean_absolute_error": skm2.mean_absolute_error, 692 "neg_mean_squared_error": skm2.mean_squared_error, 693 "neg_root_mean_squared_error": lambda x, y: np.sqrt( 694 skm2.mean_squared_error(x, y) 695 ), 696 "neg_mean_squared_log_error": skm2.mean_squared_log_error, 697 "neg_median_absolute_error": skm2.median_absolute_error, 698 "r2": skm2.r2_score, 699 } 700 701 # if p > 1: 702 # return tuple( 703 # [ 704 # scoring_options[scoring]( 705 # X_test[:, i], preds[:, i]#, **kwargs 706 # ) 707 # for i in range(p) 708 # ] 709 # ) 710 # else: 711 return scoring_options[scoring](X_test, preds)
Train on training_index, score on testing_index.
713 def plot(self, series, type_axis="dates", type_plot="pi"): 714 """Plot time series forecast 715 716 Parameters: 717 718 series: {integer} or {string} 719 series index or name 720 """ 721 722 assert all( 723 [ 724 self.mean_ is not None, 725 self.lower_ is not None, 726 self.upper_ is not None, 727 self.output_dates_ is not None, 728 ] 729 ), "model forecasting must be obtained first (with predict)" 730 731 if isinstance(series, str): 732 assert ( 733 series in self.series_names 734 ), f"series {series} doesn't exist in the input dataset" 735 series_idx = self.df_.columns.get_loc(series) 736 else: 737 assert isinstance(series, int) and ( 738 0 <= series < self.n_series 739 ), f"check series index (< {self.n_series})" 740 series_idx = series 741 742 y_all = list(self.df_.iloc[:, series_idx]) + list( 743 self.mean_.iloc[:, series_idx] 744 ) 745 y_test = list(self.mean_.iloc[:, series_idx]) 746 n_points_all = len(y_all) 747 n_points_train = self.df_.shape[0] 748 749 if type_axis == "numeric": 750 x_all = [i for i in range(n_points_all)] 751 x_test = [i for i in range(n_points_train, n_points_all)] 752 753 if type_axis == "dates": # use dates 754 x_all = np.concatenate( 755 (self.input_dates.values, self.output_dates_.values), axis=None 756 ) 757 x_test = self.output_dates_.values 758 759 if type_plot == "pi": 760 fig, ax = plt.subplots() 761 ax.plot(x_all, y_all, "-") 762 ax.plot(x_test, y_test, "-", color="orange") 763 ax.fill_between( 764 x_test, 765 self.lower_.iloc[:, series_idx], 766 self.upper_.iloc[:, series_idx], 767 alpha=0.2, 768 color="orange", 769 ) 770 plt.title( 771 f"prediction intervals for {self.replications} simulations of {series}", 772 loc="left", 773 fontsize=12, 774 fontweight=0, 775 color="black", 776 ) 777 plt.show() 778 779 if type_plot == "spaghetti": 780 palette = plt.get_cmap("Set1") 781 sims_ix = getsims(self.sims_, series_idx) 782 plt.plot(x_all, y_all, "-") 783 for col_ix in range( 784 sims_ix.shape[1] 785 ): # avoid this when there are thousands of simulations 786 plt.plot( 787 x_test, 788 sims_ix[:, col_ix], 789 "-", 790 color=palette(col_ix), 791 linewidth=1, 792 alpha=0.9, 793 ) 794 plt.plot(x_all, y_all, "-", color="black") 795 plt.plot(x_test, y_test, "-", color="blue") 796 # Add titles 797 plt.title( 798 f"{self.replications} simulations of {series}", 799 loc="left", 800 fontsize=12, 801 fontweight=0, 802 color="black", 803 ) 804 plt.xlabel("Time") 805 plt.ylabel("Values") 806 # Show the graph 807 plt.show()
Plot time series forecast
Parameters:
series: {integer} or {string}
series index or name
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
16class MultitaskClassifier(Base, ClassifierMixin): 17 """Multitask Classification model based on regression models, with shared covariates 18 19 Parameters: 20 21 obj: object 22 any object (must be a regression model) containing a method fit (obj.fit()) 23 and a method predict (obj.predict()) 24 25 n_hidden_features: int 26 number of nodes in the hidden layer 27 28 activation_name: str 29 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 30 31 a: float 32 hyperparameter for 'prelu' or 'elu' activation function 33 34 nodes_sim: str 35 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 36 'uniform' 37 38 bias: boolean 39 indicates if the hidden layer contains a bias term (True) or not 40 (False) 41 42 dropout: float 43 regularization parameter; (random) percentage of nodes dropped out 44 of the training 45 46 direct_link: boolean 47 indicates if the original predictors are included (True) in model's 48 fitting or not (False) 49 50 n_clusters: int 51 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: 52 no clustering) 53 54 cluster_encode: bool 55 defines how the variable containing clusters is treated (default is one-hot) 56 if `False`, then labels are used, without one-hot encoding 57 58 type_clust: str 59 type of clustering method: currently k-means ('kmeans') or Gaussian 60 Mixture Model ('gmm') 61 62 type_scaling: a tuple of 3 strings 63 scaling methods for inputs, hidden layer, and clustering respectively 64 (and when relevant). 65 Currently available: standardization ('std') or MinMax scaling ('minmax') 66 67 col_sample: float 68 percentage of covariates randomly chosen for training 69 70 row_sample: float 71 percentage of rows chosen for training, by stratified bootstrapping 72 73 seed: int 74 reproducibility seed for nodes_sim=='uniform' 75 76 backend: str 77 "cpu" or "gpu" or "tpu" 78 79 Attributes: 80 81 fit_objs_: dict 82 objects adjusted to each individual time series 83 84 n_classes_: int 85 number of classes for the classifier 86 87 Examples: 88 89 See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/mtask_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/mtask_classification.py) 90 91 ```python 92 import nnetsauce as ns 93 import numpy as np 94 from sklearn.datasets import load_breast_cancer 95 from sklearn.linear_model import LinearRegression 96 from sklearn.model_selection import train_test_split 97 from sklearn import metrics 98 from time import time 99 100 breast_cancer = load_breast_cancer() 101 Z = breast_cancer.data 102 t = breast_cancer.target 103 104 X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2, 105 random_state=123+2*10) 106 107 # Linear Regression is used 108 regr = LinearRegression() 109 fit_obj = ns.MultitaskClassifier(regr, n_hidden_features=5, 110 n_clusters=2, type_clust="gmm") 111 112 start = time() 113 fit_obj.fit(X_train, y_train) 114 print(f"Elapsed {time() - start}") 115 116 print(fit_obj.score(X_test, y_test)) 117 print(fit_obj.score(X_test, y_test, scoring="roc_auc")) 118 119 start = time() 120 preds = fit_obj.predict(X_test) 121 print(f"Elapsed {time() - start}") 122 print(metrics.classification_report(preds, y_test)) 123 ``` 124 125 """ 126 127 # construct the object ----- 128 129 def __init__( 130 self, 131 obj, 132 n_hidden_features=5, 133 activation_name="relu", 134 a=0.01, 135 nodes_sim="sobol", 136 bias=True, 137 dropout=0, 138 direct_link=True, 139 n_clusters=2, 140 cluster_encode=True, 141 type_clust="kmeans", 142 type_scaling=("std", "std", "std"), 143 col_sample=1, 144 row_sample=1, 145 seed=123, 146 backend="cpu", 147 ): 148 super().__init__( 149 n_hidden_features=n_hidden_features, 150 activation_name=activation_name, 151 a=a, 152 nodes_sim=nodes_sim, 153 bias=bias, 154 dropout=dropout, 155 direct_link=direct_link, 156 n_clusters=n_clusters, 157 cluster_encode=cluster_encode, 158 type_clust=type_clust, 159 type_scaling=type_scaling, 160 col_sample=col_sample, 161 row_sample=row_sample, 162 seed=seed, 163 backend=backend, 164 ) 165 166 self.type_fit = "classification" 167 self.obj = obj 168 self.fit_objs_ = {} 169 170 def fit(self, X, y, sample_weight=None, **kwargs): 171 """Fit MultitaskClassifier to training data (X, y). 172 173 Args: 174 175 X: {array-like}, shape = [n_samples, n_features] 176 Training vectors, where n_samples is the number 177 of samples and n_features is the number of features. 178 179 y: array-like, shape = [n_samples] 180 Target values. 181 182 **kwargs: additional parameters to be passed to 183 self.cook_training_set or self.obj.fit 184 185 Returns: 186 187 self: object 188 189 """ 190 191 assert mx.is_factor(y), "y must contain only integers" 192 193 output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 194 195 self.n_classes_ = len(np.unique(y)) 196 197 # multitask response 198 Y = mo.one_hot_encode2(output_y, self.n_classes_) 199 200 # if sample_weight is None: 201 for i in range(self.n_classes_): 202 self.fit_objs_[i] = pickle.loads( 203 pickle.dumps(self.obj.fit(scaled_Z, Y[:, i], **kwargs), -1) 204 ) 205 206 return self 207 208 def predict(self, X, **kwargs): 209 """Predict test data X. 210 211 Args: 212 213 X: {array-like}, shape = [n_samples, n_features] 214 Training vectors, where n_samples is the number 215 of samples and n_features is the number of features. 216 217 **kwargs: additional parameters to be passed to 218 self.cook_test_set 219 220 Returns: 221 222 model predictions: {array-like} 223 224 """ 225 226 return np.argmax(self.predict_proba(X, **kwargs), axis=1) 227 228 def predict_proba(self, X, **kwargs): 229 """Predict probabilities for test data X. 230 231 Args: 232 233 X: {array-like}, shape = [n_samples, n_features] 234 Training vectors, where n_samples is the number 235 of samples and n_features is the number of features. 236 237 **kwargs: additional parameters to be passed to 238 self.cook_test_set 239 240 Returns: 241 242 probability estimates for test data: {array-like} 243 244 """ 245 246 shape_X = X.shape 247 248 probs = np.zeros((shape_X[0], self.n_classes_)) 249 250 if len(shape_X) == 1: 251 n_features = shape_X[0] 252 253 new_X = mo.rbind( 254 X.reshape(1, n_features), 255 np.ones(n_features).reshape(1, n_features), 256 ) 257 258 Z = self.cook_test_set(new_X, **kwargs) 259 260 # loop on all the classes 261 for i in range(self.n_classes_): 262 probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0] 263 264 else: 265 Z = self.cook_test_set(X, **kwargs) 266 267 # loop on all the classes 268 for i in range(self.n_classes_): 269 probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs) 270 271 expit_raw_probs = expit(probs) 272 273 return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None] 274 275 def score(self, X, y, scoring=None, **kwargs): 276 """ Score the model on test set features X and response y. 277 278 Args: 279 280 X: {array-like}, shape = [n_samples, n_features] 281 Training vectors, where n_samples is the number 282 of samples and n_features is the number of features 283 284 y: array-like, shape = [n_samples] 285 Target values 286 287 scoring: str 288 must be in ('explained_variance', 'neg_mean_absolute_error', \ 289 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 290 'neg_median_absolute_error', 'r2') 291 292 **kwargs: additional parameters to be passed to scoring functions 293 294 Returns: 295 296 model scores: {array-like} 297 298 """ 299 300 preds = self.predict(X) 301 302 if scoring is None: 303 scoring = "accuracy" 304 305 # check inputs 306 assert scoring in ( 307 "accuracy", 308 "average_precision", 309 "brier_score_loss", 310 "f1", 311 "f1_micro", 312 "f1_macro", 313 "f1_weighted", 314 "f1_samples", 315 "neg_log_loss", 316 "precision", 317 "recall", 318 "roc_auc", 319 ), "'scoring' should be in ('accuracy', 'average_precision', \ 320 'brier_score_loss', 'f1', 'f1_micro', \ 321 'f1_macro', 'f1_weighted', 'f1_samples', \ 322 'neg_log_loss', 'precision', 'recall', \ 323 'roc_auc')" 324 325 scoring_options = { 326 "accuracy": skm2.accuracy_score, 327 "average_precision": skm2.average_precision_score, 328 "brier_score_loss": skm2.brier_score_loss, 329 "f1": skm2.f1_score, 330 "f1_micro": skm2.f1_score, 331 "f1_macro": skm2.f1_score, 332 "f1_weighted": skm2.f1_score, 333 "f1_samples": skm2.f1_score, 334 "neg_log_loss": skm2.log_loss, 335 "precision": skm2.precision_score, 336 "recall": skm2.recall_score, 337 "roc_auc": skm2.roc_auc_score, 338 } 339 340 return scoring_options[scoring](y, preds, **kwargs)
Multitask Classification model based on regression models, with shared covariates
Parameters:
obj: object
any object (must be a regression model) containing a method fit (obj.fit())
and a method predict (obj.predict())
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not
(False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original predictors are included (True) in model's
fitting or not (False)
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
col_sample: float
percentage of covariates randomly chosen for training
row_sample: float
percentage of rows chosen for training, by stratified bootstrapping
seed: int
reproducibility seed for nodes_sim=='uniform'
backend: str
"cpu" or "gpu" or "tpu"
Attributes:
fit_objs_: dict
objects adjusted to each individual time series
n_classes_: int
number of classes for the classifier
Examples:
See also https://github.com/Techtonique/nnetsauce/blob/master/examples/mtask_classification.py
import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time
breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target
X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2,
random_state=123+2*10)
# Linear Regression is used
regr = LinearRegression()
fit_obj = ns.MultitaskClassifier(regr, n_hidden_features=5,
n_clusters=2, type_clust="gmm")
start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")
print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
129 def __init__( 130 self, 131 obj, 132 n_hidden_features=5, 133 activation_name="relu", 134 a=0.01, 135 nodes_sim="sobol", 136 bias=True, 137 dropout=0, 138 direct_link=True, 139 n_clusters=2, 140 cluster_encode=True, 141 type_clust="kmeans", 142 type_scaling=("std", "std", "std"), 143 col_sample=1, 144 row_sample=1, 145 seed=123, 146 backend="cpu", 147 ): 148 super().__init__( 149 n_hidden_features=n_hidden_features, 150 activation_name=activation_name, 151 a=a, 152 nodes_sim=nodes_sim, 153 bias=bias, 154 dropout=dropout, 155 direct_link=direct_link, 156 n_clusters=n_clusters, 157 cluster_encode=cluster_encode, 158 type_clust=type_clust, 159 type_scaling=type_scaling, 160 col_sample=col_sample, 161 row_sample=row_sample, 162 seed=seed, 163 backend=backend, 164 ) 165 166 self.type_fit = "classification" 167 self.obj = obj 168 self.fit_objs_ = {}
170 def fit(self, X, y, sample_weight=None, **kwargs): 171 """Fit MultitaskClassifier to training data (X, y). 172 173 Args: 174 175 X: {array-like}, shape = [n_samples, n_features] 176 Training vectors, where n_samples is the number 177 of samples and n_features is the number of features. 178 179 y: array-like, shape = [n_samples] 180 Target values. 181 182 **kwargs: additional parameters to be passed to 183 self.cook_training_set or self.obj.fit 184 185 Returns: 186 187 self: object 188 189 """ 190 191 assert mx.is_factor(y), "y must contain only integers" 192 193 output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 194 195 self.n_classes_ = len(np.unique(y)) 196 197 # multitask response 198 Y = mo.one_hot_encode2(output_y, self.n_classes_) 199 200 # if sample_weight is None: 201 for i in range(self.n_classes_): 202 self.fit_objs_[i] = pickle.loads( 203 pickle.dumps(self.obj.fit(scaled_Z, Y[:, i], **kwargs), -1) 204 ) 205 206 return self
Fit MultitaskClassifier to training data (X, y).
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
208 def predict(self, X, **kwargs): 209 """Predict test data X. 210 211 Args: 212 213 X: {array-like}, shape = [n_samples, n_features] 214 Training vectors, where n_samples is the number 215 of samples and n_features is the number of features. 216 217 **kwargs: additional parameters to be passed to 218 self.cook_test_set 219 220 Returns: 221 222 model predictions: {array-like} 223 224 """ 225 226 return np.argmax(self.predict_proba(X, **kwargs), axis=1)
Predict test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
228 def predict_proba(self, X, **kwargs): 229 """Predict probabilities for test data X. 230 231 Args: 232 233 X: {array-like}, shape = [n_samples, n_features] 234 Training vectors, where n_samples is the number 235 of samples and n_features is the number of features. 236 237 **kwargs: additional parameters to be passed to 238 self.cook_test_set 239 240 Returns: 241 242 probability estimates for test data: {array-like} 243 244 """ 245 246 shape_X = X.shape 247 248 probs = np.zeros((shape_X[0], self.n_classes_)) 249 250 if len(shape_X) == 1: 251 n_features = shape_X[0] 252 253 new_X = mo.rbind( 254 X.reshape(1, n_features), 255 np.ones(n_features).reshape(1, n_features), 256 ) 257 258 Z = self.cook_test_set(new_X, **kwargs) 259 260 # loop on all the classes 261 for i in range(self.n_classes_): 262 probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0] 263 264 else: 265 Z = self.cook_test_set(X, **kwargs) 266 267 # loop on all the classes 268 for i in range(self.n_classes_): 269 probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs) 270 271 expit_raw_probs = expit(probs) 272 273 return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None]
Predict probabilities for test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
probability estimates for test data: {array-like}
275 def score(self, X, y, scoring=None, **kwargs): 276 """ Score the model on test set features X and response y. 277 278 Args: 279 280 X: {array-like}, shape = [n_samples, n_features] 281 Training vectors, where n_samples is the number 282 of samples and n_features is the number of features 283 284 y: array-like, shape = [n_samples] 285 Target values 286 287 scoring: str 288 must be in ('explained_variance', 'neg_mean_absolute_error', \ 289 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 290 'neg_median_absolute_error', 'r2') 291 292 **kwargs: additional parameters to be passed to scoring functions 293 294 Returns: 295 296 model scores: {array-like} 297 298 """ 299 300 preds = self.predict(X) 301 302 if scoring is None: 303 scoring = "accuracy" 304 305 # check inputs 306 assert scoring in ( 307 "accuracy", 308 "average_precision", 309 "brier_score_loss", 310 "f1", 311 "f1_micro", 312 "f1_macro", 313 "f1_weighted", 314 "f1_samples", 315 "neg_log_loss", 316 "precision", 317 "recall", 318 "roc_auc", 319 ), "'scoring' should be in ('accuracy', 'average_precision', \ 320 'brier_score_loss', 'f1', 'f1_micro', \ 321 'f1_macro', 'f1_weighted', 'f1_samples', \ 322 'neg_log_loss', 'precision', 'recall', \ 323 'roc_auc')" 324 325 scoring_options = { 326 "accuracy": skm2.accuracy_score, 327 "average_precision": skm2.average_precision_score, 328 "brier_score_loss": skm2.brier_score_loss, 329 "f1": skm2.f1_score, 330 "f1_micro": skm2.f1_score, 331 "f1_macro": skm2.f1_score, 332 "f1_weighted": skm2.f1_score, 333 "f1_samples": skm2.f1_score, 334 "neg_log_loss": skm2.log_loss, 335 "precision": skm2.precision_score, 336 "recall": skm2.recall_score, 337 "roc_auc": skm2.roc_auc_score, 338 } 339 340 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
17class SimpleMultitaskClassifier(Base, ClassifierMixin): 18 """Multitask Classification model based on regression models, with shared covariates 19 20 Parameters: 21 22 obj: object 23 any object (must be a regression model) containing a method fit (obj.fit()) 24 and a method predict (obj.predict()) 25 26 seed: int 27 reproducibility seed 28 29 Attributes: 30 31 fit_objs_: dict 32 objects adjusted to each individual time series 33 34 n_classes_: int 35 number of classes for the classifier 36 37 Examples: 38 39 ```python 40 import nnetsauce as ns 41 import numpy as np 42 from sklearn.datasets import load_breast_cancer 43 from sklearn.linear_model import LinearRegression 44 from sklearn.model_selection import train_test_split 45 from sklearn import metrics 46 from time import time 47 48 breast_cancer = load_breast_cancer() 49 Z = breast_cancer.data 50 t = breast_cancer.target 51 52 X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2, 53 random_state=123+2*10) 54 55 # Linear Regression is used 56 regr = LinearRegression() 57 fit_obj = ns.SimpleMultitaskClassifier(regr) 58 59 start = time() 60 fit_obj.fit(X_train, y_train) 61 print(f"Elapsed {time() - start}") 62 63 print(fit_obj.score(X_test, y_test)) 64 print(fit_obj.score(X_test, y_test, scoring="roc_auc")) 65 66 start = time() 67 preds = fit_obj.predict(X_test) 68 print(f"Elapsed {time() - start}") 69 print(metrics.classification_report(preds, y_test)) 70 ``` 71 72 """ 73 74 # construct the object ----- 75 76 def __init__( 77 self, 78 obj, 79 ): 80 self.type_fit = "classification" 81 self.obj = obj 82 self.fit_objs_ = {} 83 self.X_scaler_ = StandardScaler() 84 self.scaled_X_ = None 85 86 def fit(self, X, y, sample_weight=None, **kwargs): 87 """Fit SimpleMultitaskClassifier to training data (X, y). 88 89 Args: 90 91 X: {array-like}, shape = [n_samples, n_features] 92 Training vectors, where n_samples is the number 93 of samples and n_features is the number of features. 94 95 y: array-like, shape = [n_samples] 96 Target values. 97 98 **kwargs: additional parameters to be passed to 99 self.cook_training_set or self.obj.fit 100 101 Returns: 102 103 self: object 104 105 """ 106 107 assert mx.is_factor(y), "y must contain only integers" 108 109 self.scaled_X_ = self.X_scaler_.fit_transform(X) 110 111 self.n_classes_ = len(np.unique(y)) 112 113 # multitask response 114 Y = mo.one_hot_encode2(y, self.n_classes_) 115 116 # if sample_weight is None: 117 for i in range(self.n_classes_): 118 self.fit_objs_[i] = deepcopy( 119 self.obj.fit(self.scaled_X_, Y[:, i], **kwargs) 120 ) 121 122 return self 123 124 def predict(self, X, **kwargs): 125 """Predict test data X. 126 127 Args: 128 129 X: {array-like}, shape = [n_samples, n_features] 130 Training vectors, where n_samples is the number 131 of samples and n_features is the number of features. 132 133 **kwargs: additional parameters 134 135 Returns: 136 137 model predictions: {array-like} 138 139 """ 140 141 return np.argmax(self.predict_proba(X, **kwargs), axis=1) 142 143 def predict_proba(self, X, **kwargs): 144 """Predict probabilities for test data X. 145 146 Args: 147 148 X: {array-like}, shape = [n_samples, n_features] 149 Training vectors, where n_samples is the number 150 of samples and n_features is the number of features. 151 152 **kwargs: additional parameters 153 154 Returns: 155 156 probability estimates for test data: {array-like} 157 158 """ 159 160 shape_X = X.shape 161 162 probs = np.zeros((shape_X[0], self.n_classes_)) 163 164 if len(shape_X) == 1: 165 n_features = shape_X[0] 166 167 new_X = mo.rbind( 168 X.reshape(1, n_features), 169 np.ones(n_features).reshape(1, n_features), 170 ) 171 172 Z = self.X_scaler_.transform(new_X, **kwargs) 173 174 # loop on all the classes 175 for i in range(self.n_classes_): 176 probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0] 177 178 else: 179 Z = self.X_scaler_.transform(X, **kwargs) 180 181 # loop on all the classes 182 for i in range(self.n_classes_): 183 probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs) 184 185 expit_raw_probs = expit(probs) 186 187 return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None] 188 189 def score(self, X, y, scoring=None, **kwargs): 190 """ Score the model on test set features X and response y. 191 192 Args: 193 194 X: {array-like}, shape = [n_samples, n_features] 195 Training vectors, where n_samples is the number 196 of samples and n_features is the number of features 197 198 y: array-like, shape = [n_samples] 199 Target values 200 201 scoring: str 202 must be in ('explained_variance', 'neg_mean_absolute_error', \ 203 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 204 'neg_median_absolute_error', 'r2') 205 206 **kwargs: additional parameters to be passed to scoring functions 207 208 Returns: 209 210 model scores: {array-like} 211 212 """ 213 214 preds = self.predict(X) 215 216 if scoring is None: 217 scoring = "accuracy" 218 219 # check inputs 220 assert scoring in ( 221 "accuracy", 222 "average_precision", 223 "brier_score_loss", 224 "f1", 225 "f1_micro", 226 "f1_macro", 227 "f1_weighted", 228 "f1_samples", 229 "neg_log_loss", 230 "precision", 231 "recall", 232 "roc_auc", 233 ), "'scoring' should be in ('accuracy', 'average_precision', \ 234 'brier_score_loss', 'f1', 'f1_micro', \ 235 'f1_macro', 'f1_weighted', 'f1_samples', \ 236 'neg_log_loss', 'precision', 'recall', \ 237 'roc_auc')" 238 239 scoring_options = { 240 "accuracy": skm2.accuracy_score, 241 "average_precision": skm2.average_precision_score, 242 "brier_score_loss": skm2.brier_score_loss, 243 "f1": skm2.f1_score, 244 "f1_micro": skm2.f1_score, 245 "f1_macro": skm2.f1_score, 246 "f1_weighted": skm2.f1_score, 247 "f1_samples": skm2.f1_score, 248 "neg_log_loss": skm2.log_loss, 249 "precision": skm2.precision_score, 250 "recall": skm2.recall_score, 251 "roc_auc": skm2.roc_auc_score, 252 } 253 254 return scoring_options[scoring](y, preds, **kwargs)
Multitask Classification model based on regression models, with shared covariates
Parameters:
obj: object
any object (must be a regression model) containing a method fit (obj.fit())
and a method predict (obj.predict())
seed: int
reproducibility seed
Attributes:
fit_objs_: dict
objects adjusted to each individual time series
n_classes_: int
number of classes for the classifier
Examples:
import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time
breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target
X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2,
random_state=123+2*10)
# Linear Regression is used
regr = LinearRegression()
fit_obj = ns.SimpleMultitaskClassifier(regr)
start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")
print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
86 def fit(self, X, y, sample_weight=None, **kwargs): 87 """Fit SimpleMultitaskClassifier to training data (X, y). 88 89 Args: 90 91 X: {array-like}, shape = [n_samples, n_features] 92 Training vectors, where n_samples is the number 93 of samples and n_features is the number of features. 94 95 y: array-like, shape = [n_samples] 96 Target values. 97 98 **kwargs: additional parameters to be passed to 99 self.cook_training_set or self.obj.fit 100 101 Returns: 102 103 self: object 104 105 """ 106 107 assert mx.is_factor(y), "y must contain only integers" 108 109 self.scaled_X_ = self.X_scaler_.fit_transform(X) 110 111 self.n_classes_ = len(np.unique(y)) 112 113 # multitask response 114 Y = mo.one_hot_encode2(y, self.n_classes_) 115 116 # if sample_weight is None: 117 for i in range(self.n_classes_): 118 self.fit_objs_[i] = deepcopy( 119 self.obj.fit(self.scaled_X_, Y[:, i], **kwargs) 120 ) 121 122 return self
Fit SimpleMultitaskClassifier to training data (X, y).
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
124 def predict(self, X, **kwargs): 125 """Predict test data X. 126 127 Args: 128 129 X: {array-like}, shape = [n_samples, n_features] 130 Training vectors, where n_samples is the number 131 of samples and n_features is the number of features. 132 133 **kwargs: additional parameters 134 135 Returns: 136 137 model predictions: {array-like} 138 139 """ 140 141 return np.argmax(self.predict_proba(X, **kwargs), axis=1)
Predict test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters
Returns:
model predictions: {array-like}
143 def predict_proba(self, X, **kwargs): 144 """Predict probabilities for test data X. 145 146 Args: 147 148 X: {array-like}, shape = [n_samples, n_features] 149 Training vectors, where n_samples is the number 150 of samples and n_features is the number of features. 151 152 **kwargs: additional parameters 153 154 Returns: 155 156 probability estimates for test data: {array-like} 157 158 """ 159 160 shape_X = X.shape 161 162 probs = np.zeros((shape_X[0], self.n_classes_)) 163 164 if len(shape_X) == 1: 165 n_features = shape_X[0] 166 167 new_X = mo.rbind( 168 X.reshape(1, n_features), 169 np.ones(n_features).reshape(1, n_features), 170 ) 171 172 Z = self.X_scaler_.transform(new_X, **kwargs) 173 174 # loop on all the classes 175 for i in range(self.n_classes_): 176 probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs)[0] 177 178 else: 179 Z = self.X_scaler_.transform(X, **kwargs) 180 181 # loop on all the classes 182 for i in range(self.n_classes_): 183 probs[:, i] = self.fit_objs_[i].predict(Z, **kwargs) 184 185 expit_raw_probs = expit(probs) 186 187 return expit_raw_probs / expit_raw_probs.sum(axis=1)[:, None]
Predict probabilities for test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters
Returns:
probability estimates for test data: {array-like}
189 def score(self, X, y, scoring=None, **kwargs): 190 """ Score the model on test set features X and response y. 191 192 Args: 193 194 X: {array-like}, shape = [n_samples, n_features] 195 Training vectors, where n_samples is the number 196 of samples and n_features is the number of features 197 198 y: array-like, shape = [n_samples] 199 Target values 200 201 scoring: str 202 must be in ('explained_variance', 'neg_mean_absolute_error', \ 203 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 204 'neg_median_absolute_error', 'r2') 205 206 **kwargs: additional parameters to be passed to scoring functions 207 208 Returns: 209 210 model scores: {array-like} 211 212 """ 213 214 preds = self.predict(X) 215 216 if scoring is None: 217 scoring = "accuracy" 218 219 # check inputs 220 assert scoring in ( 221 "accuracy", 222 "average_precision", 223 "brier_score_loss", 224 "f1", 225 "f1_micro", 226 "f1_macro", 227 "f1_weighted", 228 "f1_samples", 229 "neg_log_loss", 230 "precision", 231 "recall", 232 "roc_auc", 233 ), "'scoring' should be in ('accuracy', 'average_precision', \ 234 'brier_score_loss', 'f1', 'f1_micro', \ 235 'f1_macro', 'f1_weighted', 'f1_samples', \ 236 'neg_log_loss', 'precision', 'recall', \ 237 'roc_auc')" 238 239 scoring_options = { 240 "accuracy": skm2.accuracy_score, 241 "average_precision": skm2.average_precision_score, 242 "brier_score_loss": skm2.brier_score_loss, 243 "f1": skm2.f1_score, 244 "f1_micro": skm2.f1_score, 245 "f1_macro": skm2.f1_score, 246 "f1_weighted": skm2.f1_score, 247 "f1_samples": skm2.f1_score, 248 "neg_log_loss": skm2.log_loss, 249 "precision": skm2.precision_score, 250 "recall": skm2.recall_score, 251 "roc_auc": skm2.roc_auc_score, 252 } 253 254 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
9class Optimizer: 10 """Optimizer class 11 12 Attributes: 13 14 type_optim: str 15 type of optimizer, (currently) either 'sgd' (stochastic minibatch gradient descent) 16 or 'scd' (stochastic minibatch coordinate descent) 17 18 num_iters: int 19 number of iterations of the optimizer 20 21 learning_rate: float 22 step size 23 24 batch_prop: float 25 proportion of the initial data used at each optimization step 26 27 learning_method: str 28 "poly" - learning rate decreasing as a polynomial function 29 of # of iterations (default) 30 "exp" - learning rate decreasing as an exponential function 31 of # of iterations 32 "momentum" - gradient descent using momentum 33 34 randomization: str 35 type of randomization applied at each step 36 "strat" - stratified subsampling (default) 37 "shuffle" - random subsampling 38 39 mass: float 40 mass on velocity, for `method` == "momentum" 41 42 decay: float 43 coefficient of decrease of the learning rate for 44 `method` == "poly" and `method` == "exp" 45 46 tolerance: float 47 early stopping parameter (convergence of loss function) 48 49 verbose: int 50 controls verbosity of gradient descent 51 0 - nothing is printed 52 1 - a progress bar is printed 53 2 - successive loss function values are printed 54 55 """ 56 57 # construct the object ----- 58 59 def __init__( 60 self, 61 type_optim="sgd", 62 num_iters=100, 63 learning_rate=0.01, 64 batch_prop=1.0, 65 learning_method="momentum", 66 randomization="strat", 67 mass=0.9, 68 decay=0.1, 69 tolerance=1e-3, 70 verbose=1, 71 ): 72 self.type_optim = type_optim 73 self.num_iters = num_iters 74 self.learning_rate = learning_rate 75 self.batch_prop = batch_prop 76 self.learning_method = learning_method 77 self.randomization = randomization 78 self.mass = mass 79 self.decay = decay 80 self.tolerance = tolerance 81 self.verbose = verbose 82 self.opt = None 83 84 def fit(self, loss_func, response, x0, **kwargs): 85 """Fit GLM model to training data (X, y). 86 87 Args: 88 89 loss_func: loss function 90 91 response: array-like, shape = [n_samples] 92 target variable (used for subsampling) 93 94 x0: array-like, shape = [n_features] 95 initial value provided to the optimizer 96 97 **kwargs: additional parameters to be passed to 98 loss function 99 100 Returns: 101 102 self: object 103 104 """ 105 106 if self.type_optim == "scd": 107 self.results = scd( 108 loss_func, 109 response=response, 110 x=x0, 111 num_iters=self.num_iters, 112 batch_prop=self.batch_prop, 113 learning_rate=self.learning_rate, 114 learning_method=self.learning_method, 115 mass=self.mass, 116 decay=self.decay, 117 randomization=self.randomization, 118 tolerance=self.tolerance, 119 verbose=self.verbose, 120 **kwargs 121 ) 122 123 if self.type_optim == "sgd": 124 self.results = sgd( 125 loss_func, 126 response=response, 127 x=x0, 128 num_iters=self.num_iters, 129 batch_prop=self.batch_prop, 130 learning_rate=self.learning_rate, 131 learning_method=self.learning_method, 132 mass=self.mass, 133 decay=self.decay, 134 randomization=self.randomization, 135 tolerance=self.tolerance, 136 verbose=self.verbose, 137 **kwargs 138 ) 139 140 return self 141 142 def one_hot_encode(self, y, n_classes): 143 return one_hot_encode(y, n_classes)
Optimizer class
Attributes:
type_optim: str
type of optimizer, (currently) either 'sgd' (stochastic minibatch gradient descent)
or 'scd' (stochastic minibatch coordinate descent)
num_iters: int
number of iterations of the optimizer
learning_rate: float
step size
batch_prop: float
proportion of the initial data used at each optimization step
learning_method: str
"poly" - learning rate decreasing as a polynomial function
of # of iterations (default)
"exp" - learning rate decreasing as an exponential function
of # of iterations
"momentum" - gradient descent using momentum
randomization: str
type of randomization applied at each step
"strat" - stratified subsampling (default)
"shuffle" - random subsampling
mass: float
mass on velocity, for `method` == "momentum"
decay: float
coefficient of decrease of the learning rate for
`method` == "poly" and `method` == "exp"
tolerance: float
early stopping parameter (convergence of loss function)
verbose: int
controls verbosity of gradient descent
0 - nothing is printed
1 - a progress bar is printed
2 - successive loss function values are printed
59 def __init__( 60 self, 61 type_optim="sgd", 62 num_iters=100, 63 learning_rate=0.01, 64 batch_prop=1.0, 65 learning_method="momentum", 66 randomization="strat", 67 mass=0.9, 68 decay=0.1, 69 tolerance=1e-3, 70 verbose=1, 71 ): 72 self.type_optim = type_optim 73 self.num_iters = num_iters 74 self.learning_rate = learning_rate 75 self.batch_prop = batch_prop 76 self.learning_method = learning_method 77 self.randomization = randomization 78 self.mass = mass 79 self.decay = decay 80 self.tolerance = tolerance 81 self.verbose = verbose 82 self.opt = None
84 def fit(self, loss_func, response, x0, **kwargs): 85 """Fit GLM model to training data (X, y). 86 87 Args: 88 89 loss_func: loss function 90 91 response: array-like, shape = [n_samples] 92 target variable (used for subsampling) 93 94 x0: array-like, shape = [n_features] 95 initial value provided to the optimizer 96 97 **kwargs: additional parameters to be passed to 98 loss function 99 100 Returns: 101 102 self: object 103 104 """ 105 106 if self.type_optim == "scd": 107 self.results = scd( 108 loss_func, 109 response=response, 110 x=x0, 111 num_iters=self.num_iters, 112 batch_prop=self.batch_prop, 113 learning_rate=self.learning_rate, 114 learning_method=self.learning_method, 115 mass=self.mass, 116 decay=self.decay, 117 randomization=self.randomization, 118 tolerance=self.tolerance, 119 verbose=self.verbose, 120 **kwargs 121 ) 122 123 if self.type_optim == "sgd": 124 self.results = sgd( 125 loss_func, 126 response=response, 127 x=x0, 128 num_iters=self.num_iters, 129 batch_prop=self.batch_prop, 130 learning_rate=self.learning_rate, 131 learning_method=self.learning_method, 132 mass=self.mass, 133 decay=self.decay, 134 randomization=self.randomization, 135 tolerance=self.tolerance, 136 verbose=self.verbose, 137 **kwargs 138 ) 139 140 return self
Fit GLM model to training data (X, y).
Args:
loss_func: loss function
response: array-like, shape = [n_samples]
target variable (used for subsampling)
x0: array-like, shape = [n_features]
initial value provided to the optimizer
**kwargs: additional parameters to be passed to
loss function
Returns:
self: object
18class RandomBagRegressor(RandomBag, RegressorMixin): 19 """Randomized 'Bagging' Regression model 20 21 Parameters: 22 23 obj: object 24 any object containing a method fit (obj.fit()) and a method predict 25 (obj.predict()) 26 27 n_estimators: int 28 number of boosting iterations 29 30 n_hidden_features: int 31 number of nodes in the hidden layer 32 33 activation_name: str 34 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 35 36 a: float 37 hyperparameter for 'prelu' or 'elu' activation function 38 39 nodes_sim: str 40 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 41 'uniform' 42 43 bias: boolean 44 indicates if the hidden layer contains a bias term (True) or not 45 (False) 46 47 dropout: float 48 regularization parameter; (random) percentage of nodes dropped out 49 of the training 50 51 direct_link: boolean 52 indicates if the original predictors are included (True) in model''s 53 fitting or not (False) 54 55 n_clusters: int 56 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: 57 no clustering) 58 59 cluster_encode: bool 60 defines how the variable containing clusters is treated (default is one-hot) 61 if `False`, then labels are used, without one-hot encoding 62 63 type_clust: str 64 type of clustering method: currently k-means ('kmeans') or Gaussian 65 Mixture Model ('gmm') 66 67 type_scaling: a tuple of 3 strings 68 scaling methods for inputs, hidden layer, and clustering respectively 69 (and when relevant). 70 Currently available: standardization ('std') or MinMax scaling ('minmax') 71 72 col_sample: float 73 percentage of covariates randomly chosen for training 74 75 row_sample: float 76 percentage of rows chosen for training, by stratified bootstrapping 77 78 seed: int 79 reproducibility seed for nodes_sim=='uniform' 80 81 backend: str 82 "cpu" or "gpu" or "tpu" 83 84 Attributes: 85 86 voter_: dict 87 dictionary containing all the fitted base-learners 88 89 90 Examples: 91 92 ```python 93 import numpy as np 94 import nnetsauce as ns 95 from sklearn.datasets import fetch_california_housing 96 from sklearn.tree import DecisionTreeRegressor 97 from sklearn.model_selection import train_test_split 98 99 X, y = fetch_california_housing(return_X_y=True, as_frame=False) 100 101 # split data into training test and test set 102 X_train, X_test, y_train, y_test = train_test_split(X, y, 103 test_size=0.2, random_state=13) 104 105 # Requires further tuning 106 obj = DecisionTreeRegressor(max_depth=3, random_state=123) 107 obj2 = ns.RandomBagRegressor(obj=obj, direct_link=False, 108 n_estimators=50, 109 col_sample=0.9, row_sample=0.9, 110 dropout=0, n_clusters=0, verbose=1) 111 112 obj2.fit(X_train, y_train) 113 114 print(np.sqrt(obj2.score(X_test, y_test))) # RMSE 115 116 ``` 117 118 """ 119 120 # construct the object ----- 121 122 def __init__( 123 self, 124 obj, 125 n_estimators=10, 126 n_hidden_features=1, 127 activation_name="relu", 128 a=0.01, 129 nodes_sim="sobol", 130 bias=True, 131 dropout=0, 132 direct_link=False, 133 n_clusters=2, 134 cluster_encode=True, 135 type_clust="kmeans", 136 type_scaling=("std", "std", "std"), 137 col_sample=1, 138 row_sample=1, 139 n_jobs=None, 140 seed=123, 141 verbose=1, 142 backend="cpu", 143 ): 144 super().__init__( 145 obj=obj, 146 n_estimators=n_estimators, 147 n_hidden_features=n_hidden_features, 148 activation_name=activation_name, 149 a=a, 150 nodes_sim=nodes_sim, 151 bias=bias, 152 dropout=dropout, 153 direct_link=direct_link, 154 n_clusters=n_clusters, 155 cluster_encode=cluster_encode, 156 type_clust=type_clust, 157 type_scaling=type_scaling, 158 col_sample=col_sample, 159 row_sample=row_sample, 160 seed=seed, 161 backend=backend, 162 ) 163 164 self.type_fit = "regression" 165 self.verbose = verbose 166 self.n_jobs = n_jobs 167 self.voter_ = {} 168 169 def fit(self, X, y, **kwargs): 170 """Fit Random 'Bagging' model to training data (X, y). 171 172 Args: 173 174 X: {array-like}, shape = [n_samples, n_features] 175 Training vectors, where n_samples is the number 176 of samples and n_features is the number of features. 177 178 y: array-like, shape = [n_samples] 179 Target values. 180 181 **kwargs: additional parameters to be passed to 182 self.cook_training_set or self.obj.fit 183 184 Returns: 185 186 self: object 187 188 """ 189 190 base_learner = CustomRegressor( 191 self.obj, 192 n_hidden_features=self.n_hidden_features, 193 activation_name=self.activation_name, 194 a=self.a, 195 nodes_sim=self.nodes_sim, 196 bias=self.bias, 197 dropout=self.dropout, 198 direct_link=self.direct_link, 199 n_clusters=self.n_clusters, 200 type_clust=self.type_clust, 201 type_scaling=self.type_scaling, 202 col_sample=self.col_sample, 203 row_sample=self.row_sample, 204 seed=self.seed, 205 ) 206 207 # 1 - Sequential training ----- 208 209 if self.n_jobs is None: 210 self.voter_ = rbagloop_regression( 211 base_learner, X, y, self.n_estimators, self.verbose, self.seed 212 ) 213 214 self.n_estimators = len(self.voter_) 215 216 return self 217 218 # 2 - Parallel training ----- 219 # buggy 220 # if self.n_jobs is not None: 221 def fit_estimators(m): 222 base_learner__ = pickle.loads(pickle.dumps(base_learner, -1)) 223 base_learner__.set_params(seed=self.seed + m * 1000) 224 base_learner__.fit(X, y, **kwargs) 225 return base_learner__ 226 227 if self.verbose == 1: 228 voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")( 229 delayed(fit_estimators)(m) 230 for m in tqdm(range(self.n_estimators)) 231 ) 232 else: 233 voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")( 234 delayed(fit_estimators)(m) for m in range(self.n_estimators) 235 ) 236 237 self.voter_ = {i: elt for i, elt in enumerate(voters_list)} 238 239 self.n_estimators = len(self.voter_) 240 241 return self 242 243 def predict(self, X, weights=None, **kwargs): 244 """Predict for test data X. 245 246 Args: 247 248 X: {array-like}, shape = [n_samples, n_features] 249 Training vectors, where n_samples is the number 250 of samples and n_features is the number of features. 251 252 **kwargs: additional parameters to be passed to 253 self.cook_test_set 254 255 Returns: 256 257 estimates for test data: {array-like} 258 259 """ 260 261 def calculate_preds(voter, weights=None): 262 ensemble_preds = 0 263 264 n_iter = len(voter) 265 266 assert n_iter > 0, "no estimator found in `RandomBag` ensemble" 267 268 if weights is None: 269 for idx, elt in voter.items(): 270 ensemble_preds += elt.predict(X) 271 272 return ensemble_preds / n_iter 273 274 # if weights is not None: 275 for idx, elt in voter.items(): 276 ensemble_preds += weights[idx] * elt.predict(X) 277 278 return ensemble_preds 279 280 # end calculate_preds ---- 281 282 if weights is None: 283 return calculate_preds(self.voter_) 284 285 # if weights is not None: 286 self.weights = weights 287 288 return calculate_preds(self.voter_, weights) 289 290 def score(self, X, y, scoring=None, **kwargs): 291 """ Score the model on test set features X and response y. 292 293 Args: 294 295 X: {array-like}, shape = [n_samples, n_features] 296 Training vectors, where n_samples is the number 297 of samples and n_features is the number of features 298 299 y: array-like, shape = [n_samples] 300 Target values 301 302 scoring: str 303 must be in ('explained_variance', 'neg_mean_absolute_error', \ 304 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 305 'neg_median_absolute_error', 'r2') 306 307 **kwargs: additional parameters to be passed to scoring functions 308 309 Returns: 310 311 model scores: {array-like} 312 313 """ 314 315 preds = self.predict(X) 316 317 if type(preds) == tuple: # if there are std. devs in the predictions 318 preds = preds[0] 319 320 if scoring is None: 321 scoring = "neg_mean_squared_error" 322 323 # check inputs 324 assert scoring in ( 325 "explained_variance", 326 "neg_mean_absolute_error", 327 "neg_mean_squared_error", 328 "neg_mean_squared_log_error", 329 "neg_median_absolute_error", 330 "r2", 331 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 332 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 333 'neg_median_absolute_error', 'r2')" 334 335 scoring_options = { 336 "explained_variance": skm.explained_variance_score, 337 "neg_mean_absolute_error": skm.median_absolute_error, 338 "neg_mean_squared_error": skm.mean_squared_error, 339 "neg_mean_squared_log_error": skm.mean_squared_log_error, 340 "neg_median_absolute_error": skm.median_absolute_error, 341 "r2": skm.r2_score, 342 } 343 344 return scoring_options[scoring](y, preds, **kwargs)
Randomized 'Bagging' Regression model
Parameters:
obj: object
any object containing a method fit (obj.fit()) and a method predict
(obj.predict())
n_estimators: int
number of boosting iterations
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not
(False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original predictors are included (True) in model''s
fitting or not (False)
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
col_sample: float
percentage of covariates randomly chosen for training
row_sample: float
percentage of rows chosen for training, by stratified bootstrapping
seed: int
reproducibility seed for nodes_sim=='uniform'
backend: str
"cpu" or "gpu" or "tpu"
Attributes:
voter_: dict
dictionary containing all the fitted base-learners
Examples:
import numpy as np
import nnetsauce as ns
from sklearn.datasets import fetch_california_housing
from sklearn.tree import DecisionTreeRegressor
from sklearn.model_selection import train_test_split
X, y = fetch_california_housing(return_X_y=True, as_frame=False)
# split data into training test and test set
X_train, X_test, y_train, y_test = train_test_split(X, y,
test_size=0.2, random_state=13)
# Requires further tuning
obj = DecisionTreeRegressor(max_depth=3, random_state=123)
obj2 = ns.RandomBagRegressor(obj=obj, direct_link=False,
n_estimators=50,
col_sample=0.9, row_sample=0.9,
dropout=0, n_clusters=0, verbose=1)
obj2.fit(X_train, y_train)
print(np.sqrt(obj2.score(X_test, y_test))) # RMSE
122 def __init__( 123 self, 124 obj, 125 n_estimators=10, 126 n_hidden_features=1, 127 activation_name="relu", 128 a=0.01, 129 nodes_sim="sobol", 130 bias=True, 131 dropout=0, 132 direct_link=False, 133 n_clusters=2, 134 cluster_encode=True, 135 type_clust="kmeans", 136 type_scaling=("std", "std", "std"), 137 col_sample=1, 138 row_sample=1, 139 n_jobs=None, 140 seed=123, 141 verbose=1, 142 backend="cpu", 143 ): 144 super().__init__( 145 obj=obj, 146 n_estimators=n_estimators, 147 n_hidden_features=n_hidden_features, 148 activation_name=activation_name, 149 a=a, 150 nodes_sim=nodes_sim, 151 bias=bias, 152 dropout=dropout, 153 direct_link=direct_link, 154 n_clusters=n_clusters, 155 cluster_encode=cluster_encode, 156 type_clust=type_clust, 157 type_scaling=type_scaling, 158 col_sample=col_sample, 159 row_sample=row_sample, 160 seed=seed, 161 backend=backend, 162 ) 163 164 self.type_fit = "regression" 165 self.verbose = verbose 166 self.n_jobs = n_jobs 167 self.voter_ = {}
169 def fit(self, X, y, **kwargs): 170 """Fit Random 'Bagging' model to training data (X, y). 171 172 Args: 173 174 X: {array-like}, shape = [n_samples, n_features] 175 Training vectors, where n_samples is the number 176 of samples and n_features is the number of features. 177 178 y: array-like, shape = [n_samples] 179 Target values. 180 181 **kwargs: additional parameters to be passed to 182 self.cook_training_set or self.obj.fit 183 184 Returns: 185 186 self: object 187 188 """ 189 190 base_learner = CustomRegressor( 191 self.obj, 192 n_hidden_features=self.n_hidden_features, 193 activation_name=self.activation_name, 194 a=self.a, 195 nodes_sim=self.nodes_sim, 196 bias=self.bias, 197 dropout=self.dropout, 198 direct_link=self.direct_link, 199 n_clusters=self.n_clusters, 200 type_clust=self.type_clust, 201 type_scaling=self.type_scaling, 202 col_sample=self.col_sample, 203 row_sample=self.row_sample, 204 seed=self.seed, 205 ) 206 207 # 1 - Sequential training ----- 208 209 if self.n_jobs is None: 210 self.voter_ = rbagloop_regression( 211 base_learner, X, y, self.n_estimators, self.verbose, self.seed 212 ) 213 214 self.n_estimators = len(self.voter_) 215 216 return self 217 218 # 2 - Parallel training ----- 219 # buggy 220 # if self.n_jobs is not None: 221 def fit_estimators(m): 222 base_learner__ = pickle.loads(pickle.dumps(base_learner, -1)) 223 base_learner__.set_params(seed=self.seed + m * 1000) 224 base_learner__.fit(X, y, **kwargs) 225 return base_learner__ 226 227 if self.verbose == 1: 228 voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")( 229 delayed(fit_estimators)(m) 230 for m in tqdm(range(self.n_estimators)) 231 ) 232 else: 233 voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")( 234 delayed(fit_estimators)(m) for m in range(self.n_estimators) 235 ) 236 237 self.voter_ = {i: elt for i, elt in enumerate(voters_list)} 238 239 self.n_estimators = len(self.voter_) 240 241 return self
Fit Random 'Bagging' model to training data (X, y).
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
243 def predict(self, X, weights=None, **kwargs): 244 """Predict for test data X. 245 246 Args: 247 248 X: {array-like}, shape = [n_samples, n_features] 249 Training vectors, where n_samples is the number 250 of samples and n_features is the number of features. 251 252 **kwargs: additional parameters to be passed to 253 self.cook_test_set 254 255 Returns: 256 257 estimates for test data: {array-like} 258 259 """ 260 261 def calculate_preds(voter, weights=None): 262 ensemble_preds = 0 263 264 n_iter = len(voter) 265 266 assert n_iter > 0, "no estimator found in `RandomBag` ensemble" 267 268 if weights is None: 269 for idx, elt in voter.items(): 270 ensemble_preds += elt.predict(X) 271 272 return ensemble_preds / n_iter 273 274 # if weights is not None: 275 for idx, elt in voter.items(): 276 ensemble_preds += weights[idx] * elt.predict(X) 277 278 return ensemble_preds 279 280 # end calculate_preds ---- 281 282 if weights is None: 283 return calculate_preds(self.voter_) 284 285 # if weights is not None: 286 self.weights = weights 287 288 return calculate_preds(self.voter_, weights)
Predict for test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
estimates for test data: {array-like}
290 def score(self, X, y, scoring=None, **kwargs): 291 """ Score the model on test set features X and response y. 292 293 Args: 294 295 X: {array-like}, shape = [n_samples, n_features] 296 Training vectors, where n_samples is the number 297 of samples and n_features is the number of features 298 299 y: array-like, shape = [n_samples] 300 Target values 301 302 scoring: str 303 must be in ('explained_variance', 'neg_mean_absolute_error', \ 304 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 305 'neg_median_absolute_error', 'r2') 306 307 **kwargs: additional parameters to be passed to scoring functions 308 309 Returns: 310 311 model scores: {array-like} 312 313 """ 314 315 preds = self.predict(X) 316 317 if type(preds) == tuple: # if there are std. devs in the predictions 318 preds = preds[0] 319 320 if scoring is None: 321 scoring = "neg_mean_squared_error" 322 323 # check inputs 324 assert scoring in ( 325 "explained_variance", 326 "neg_mean_absolute_error", 327 "neg_mean_squared_error", 328 "neg_mean_squared_log_error", 329 "neg_median_absolute_error", 330 "r2", 331 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 332 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 333 'neg_median_absolute_error', 'r2')" 334 335 scoring_options = { 336 "explained_variance": skm.explained_variance_score, 337 "neg_mean_absolute_error": skm.median_absolute_error, 338 "neg_mean_squared_error": skm.mean_squared_error, 339 "neg_mean_squared_log_error": skm.mean_squared_log_error, 340 "neg_median_absolute_error": skm.median_absolute_error, 341 "r2": skm.r2_score, 342 } 343 344 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.randombag.bag.RandomBag
- obj
- n_estimators
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
18class RandomBagClassifier(RandomBag, ClassifierMixin): 19 """Randomized 'Bagging' Classification model 20 21 Parameters: 22 23 obj: object 24 any object containing a method fit (obj.fit()) and a method predict 25 (obj.predict()) 26 27 n_estimators: int 28 number of boosting iterations 29 30 n_hidden_features: int 31 number of nodes in the hidden layer 32 33 activation_name: str 34 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 35 36 a: float 37 hyperparameter for 'prelu' or 'elu' activation function 38 39 nodes_sim: str 40 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 41 'uniform' 42 43 bias: boolean 44 indicates if the hidden layer contains a bias term (True) or not 45 (False) 46 47 dropout: float 48 regularization parameter; (random) percentage of nodes dropped out 49 of the training 50 51 direct_link: boolean 52 indicates if the original predictors are included (True) in model's 53 fitting or not (False) 54 55 n_clusters: int 56 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: 57 no clustering) 58 59 cluster_encode: bool 60 defines how the variable containing clusters is treated (default is one-hot) 61 if `False`, then labels are used, without one-hot encoding 62 63 type_clust: str 64 type of clustering method: currently k-means ('kmeans') or Gaussian 65 Mixture Model ('gmm') 66 67 type_scaling: a tuple of 3 strings 68 scaling methods for inputs, hidden layer, and clustering respectively 69 (and when relevant). 70 Currently available: standardization ('std') or MinMax scaling ('minmax') 71 72 col_sample: float 73 percentage of covariates randomly chosen for training 74 75 row_sample: float 76 percentage of rows chosen for training, by stratified bootstrapping 77 78 seed: int 79 reproducibility seed for nodes_sim=='uniform' 80 81 backend: str 82 "cpu" or "gpu" or "tpu" 83 84 Attributes: 85 86 voter_: dict 87 dictionary containing all the fitted base-learners 88 89 90 Examples: 91 92 See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/randombag_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/randombag_classification.py) 93 94 ```python 95 import nnetsauce as ns 96 from sklearn.datasets import load_breast_cancer 97 from sklearn.tree import DecisionTreeClassifier 98 from sklearn.model_selection import train_test_split 99 from sklearn import metrics 100 from time import time 101 102 103 breast_cancer = load_breast_cancer() 104 Z = breast_cancer.data 105 t = breast_cancer.target 106 np.random.seed(123) 107 X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2) 108 109 # decision tree 110 clf = DecisionTreeClassifier(max_depth=2, random_state=123) 111 fit_obj = ns.RandomBagClassifier(clf, n_hidden_features=2, 112 direct_link=True, 113 n_estimators=100, 114 col_sample=0.9, row_sample=0.9, 115 dropout=0.3, n_clusters=0, verbose=1) 116 117 start = time() 118 fit_obj.fit(X_train, y_train) 119 print(f"Elapsed {time() - start}") 120 121 print(fit_obj.score(X_test, y_test)) 122 print(fit_obj.score(X_test, y_test, scoring="roc_auc")) 123 124 start = time() 125 preds = fit_obj.predict(X_test) 126 print(f"Elapsed {time() - start}") 127 print(metrics.classification_report(preds, y_test)) 128 ``` 129 130 """ 131 132 # construct the object ----- 133 134 def __init__( 135 self, 136 obj, 137 n_estimators=10, 138 n_hidden_features=1, 139 activation_name="relu", 140 a=0.01, 141 nodes_sim="sobol", 142 bias=True, 143 dropout=0, 144 direct_link=False, 145 n_clusters=2, 146 cluster_encode=True, 147 type_clust="kmeans", 148 type_scaling=("std", "std", "std"), 149 col_sample=1, 150 row_sample=1, 151 n_jobs=None, 152 seed=123, 153 verbose=1, 154 backend="cpu", 155 ): 156 super().__init__( 157 obj=obj, 158 n_estimators=n_estimators, 159 n_hidden_features=n_hidden_features, 160 activation_name=activation_name, 161 a=a, 162 nodes_sim=nodes_sim, 163 bias=bias, 164 dropout=dropout, 165 direct_link=direct_link, 166 n_clusters=n_clusters, 167 cluster_encode=cluster_encode, 168 type_clust=type_clust, 169 type_scaling=type_scaling, 170 col_sample=col_sample, 171 row_sample=row_sample, 172 seed=seed, 173 backend=backend, 174 ) 175 176 self.type_fit = "classification" 177 self.verbose = verbose 178 self.n_jobs = n_jobs 179 self.voter_ = {} 180 181 def fit(self, X, y, **kwargs): 182 """Fit Random 'Bagging' model to training data (X, y). 183 184 Args: 185 186 X: {array-like}, shape = [n_samples, n_features] 187 Training vectors, where n_samples is the number 188 of samples and n_features is the number of features. 189 190 y: array-like, shape = [n_samples] 191 Target values. 192 193 **kwargs: additional parameters to be passed to 194 self.cook_training_set or self.obj.fit 195 196 Returns: 197 198 self: object 199 200 """ 201 202 assert mx.is_factor(y), "y must contain only integers" 203 204 # training 205 self.n_classes = len(np.unique(y)) 206 207 base_learner = CustomClassifier( 208 self.obj, 209 n_hidden_features=self.n_hidden_features, 210 activation_name=self.activation_name, 211 a=self.a, 212 nodes_sim=self.nodes_sim, 213 bias=self.bias, 214 dropout=self.dropout, 215 direct_link=self.direct_link, 216 n_clusters=self.n_clusters, 217 type_clust=self.type_clust, 218 type_scaling=self.type_scaling, 219 col_sample=self.col_sample, 220 row_sample=self.row_sample, 221 seed=self.seed, 222 ) 223 224 # 1 - Sequential training ----- 225 226 if self.n_jobs is None: 227 self.voter_ = rbagloop_classification( 228 base_learner, X, y, self.n_estimators, self.verbose, self.seed 229 ) 230 231 self.n_estimators = len(self.voter_) 232 233 return self 234 235 # 2 - Parallel training ----- 236 # buggy 237 # if self.n_jobs is not None: 238 def fit_estimators(m): 239 base_learner__ = pickle.loads(pickle.dumps(base_learner, -1)) 240 base_learner__.set_params(seed=self.seed + m * 1000) 241 base_learner__.fit(X, y, **kwargs) 242 return base_learner__ 243 244 if self.verbose == 1: 245 voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")( 246 delayed(fit_estimators)(m) 247 for m in tqdm(range(self.n_estimators)) 248 ) 249 else: 250 voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")( 251 delayed(fit_estimators)(m) for m in range(self.n_estimators) 252 ) 253 254 self.voter_ = {idx: elt for idx, elt in enumerate(voters_list)} 255 256 self.n_estimators = len(self.voter_) 257 258 return self 259 260 def predict(self, X, weights=None, **kwargs): 261 """Predict test data X. 262 263 Args: 264 265 X: {array-like}, shape = [n_samples, n_features] 266 Training vectors, where n_samples is the number 267 of samples and n_features is the number of features. 268 269 **kwargs: additional parameters to be passed to 270 self.cook_test_set 271 272 Returns: 273 274 model predictions: {array-like} 275 276 """ 277 return self.predict_proba(X, weights, **kwargs).argmax(axis=1) 278 279 def predict_proba(self, X, weights=None, **kwargs): 280 """Predict probabilities for test data X. 281 282 Args: 283 284 X: {array-like}, shape = [n_samples, n_features] 285 Training vectors, where n_samples is the number 286 of samples and n_features is the number of features. 287 288 **kwargs: additional parameters to be passed to 289 self.cook_test_set 290 291 Returns: 292 293 probability estimates for test data: {array-like} 294 295 """ 296 297 def calculate_probas(voter, weights=None, verbose=None): 298 ensemble_proba = 0 299 300 n_iter = len(voter) 301 302 assert n_iter > 0, "no estimator found in `RandomBag` ensemble" 303 304 if weights is None: 305 for idx, elt in voter.items(): 306 try: 307 ensemble_proba += elt.predict_proba(X) 308 309 # if verbose == 1: 310 # pbar.update(idx) 311 312 except: 313 continue 314 315 # if verbose == 1: 316 # pbar.update(n_iter) 317 318 return ensemble_proba / n_iter 319 320 # if weights is not None: 321 for idx, elt in voter.items(): 322 ensemble_proba += weights[idx] * elt.predict_proba(X) 323 324 # if verbose == 1: 325 # pbar.update(idx) 326 327 # if verbose == 1: 328 # pbar.update(n_iter) 329 330 return ensemble_proba 331 332 # end calculate_probas ---- 333 334 if self.n_jobs is None: 335 # if self.verbose == 1: 336 # pbar = Progbar(self.n_estimators) 337 338 if weights is None: 339 return calculate_probas(self.voter_, verbose=self.verbose) 340 341 # if weights is not None: 342 self.weights = weights 343 344 return calculate_probas(self.voter_, weights, verbose=self.verbose) 345 346 # if self.n_jobs is not None: 347 def predict_estimator(m): 348 try: 349 return self.voter_[m].predict_proba(X) 350 except: 351 pass 352 353 if self.verbose == 1: 354 preds = Parallel(n_jobs=self.n_jobs, prefer="threads")( 355 delayed(predict_estimator)(m) 356 for m in tqdm(range(self.n_estimators)) 357 ) 358 359 else: 360 preds = Parallel(n_jobs=self.n_jobs, prefer="threads")( 361 delayed(predict_estimator)(m) for m in range(self.n_estimators) 362 ) 363 364 ensemble_proba = 0 365 366 if weights is None: 367 for i in range(self.n_estimators): 368 ensemble_proba += preds[i] 369 370 return ensemble_proba / self.n_estimators 371 372 for i in range(self.n_estimators): 373 ensemble_proba += weights[i] * preds[i] 374 375 return ensemble_proba 376 377 def score(self, X, y, weights=None, scoring=None, **kwargs): 378 """ Score the model on test set features X and response y. 379 380 Args: 381 382 X: {array-like}, shape = [n_samples, n_features] 383 Training vectors, where n_samples is the number 384 of samples and n_features is the number of features 385 386 y: array-like, shape = [n_samples] 387 Target values 388 389 scoring: str 390 must be in ('explained_variance', 'neg_mean_absolute_error', \ 391 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 392 'neg_median_absolute_error', 'r2') 393 394 **kwargs: additional parameters to be passed to scoring functions 395 396 Returns: 397 398 model scores: {array-like} 399 400 """ 401 402 preds = self.predict(X, weights, **kwargs) 403 404 if scoring is None: 405 scoring = "accuracy" 406 407 # check inputs 408 assert scoring in ( 409 "accuracy", 410 "average_precision", 411 "brier_score_loss", 412 "f1", 413 "f1_micro", 414 "f1_macro", 415 "f1_weighted", 416 "f1_samples", 417 "neg_log_loss", 418 "precision", 419 "recall", 420 "roc_auc", 421 ), "'scoring' should be in ('accuracy', 'average_precision', \ 422 'brier_score_loss', 'f1', 'f1_micro', \ 423 'f1_macro', 'f1_weighted', 'f1_samples', \ 424 'neg_log_loss', 'precision', 'recall', \ 425 'roc_auc')" 426 427 scoring_options = { 428 "accuracy": skm2.accuracy_score, 429 "average_precision": skm2.average_precision_score, 430 "brier_score_loss": skm2.brier_score_loss, 431 "f1": skm2.f1_score, 432 "f1_micro": skm2.f1_score, 433 "f1_macro": skm2.f1_score, 434 "f1_weighted": skm2.f1_score, 435 "f1_samples": skm2.f1_score, 436 "neg_log_loss": skm2.log_loss, 437 "precision": skm2.precision_score, 438 "recall": skm2.recall_score, 439 "roc_auc": skm2.roc_auc_score, 440 } 441 442 return scoring_options[scoring](y, preds, **kwargs)
Randomized 'Bagging' Classification model
Parameters:
obj: object
any object containing a method fit (obj.fit()) and a method predict
(obj.predict())
n_estimators: int
number of boosting iterations
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not
(False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original predictors are included (True) in model's
fitting or not (False)
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
col_sample: float
percentage of covariates randomly chosen for training
row_sample: float
percentage of rows chosen for training, by stratified bootstrapping
seed: int
reproducibility seed for nodes_sim=='uniform'
backend: str
"cpu" or "gpu" or "tpu"
Attributes:
voter_: dict
dictionary containing all the fitted base-learners
Examples:
See also https://github.com/Techtonique/nnetsauce/blob/master/examples/randombag_classification.py
import nnetsauce as ns
from sklearn.datasets import load_breast_cancer
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time
breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
# decision tree
clf = DecisionTreeClassifier(max_depth=2, random_state=123)
fit_obj = ns.RandomBagClassifier(clf, n_hidden_features=2,
direct_link=True,
n_estimators=100,
col_sample=0.9, row_sample=0.9,
dropout=0.3, n_clusters=0, verbose=1)
start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")
print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
134 def __init__( 135 self, 136 obj, 137 n_estimators=10, 138 n_hidden_features=1, 139 activation_name="relu", 140 a=0.01, 141 nodes_sim="sobol", 142 bias=True, 143 dropout=0, 144 direct_link=False, 145 n_clusters=2, 146 cluster_encode=True, 147 type_clust="kmeans", 148 type_scaling=("std", "std", "std"), 149 col_sample=1, 150 row_sample=1, 151 n_jobs=None, 152 seed=123, 153 verbose=1, 154 backend="cpu", 155 ): 156 super().__init__( 157 obj=obj, 158 n_estimators=n_estimators, 159 n_hidden_features=n_hidden_features, 160 activation_name=activation_name, 161 a=a, 162 nodes_sim=nodes_sim, 163 bias=bias, 164 dropout=dropout, 165 direct_link=direct_link, 166 n_clusters=n_clusters, 167 cluster_encode=cluster_encode, 168 type_clust=type_clust, 169 type_scaling=type_scaling, 170 col_sample=col_sample, 171 row_sample=row_sample, 172 seed=seed, 173 backend=backend, 174 ) 175 176 self.type_fit = "classification" 177 self.verbose = verbose 178 self.n_jobs = n_jobs 179 self.voter_ = {}
181 def fit(self, X, y, **kwargs): 182 """Fit Random 'Bagging' model to training data (X, y). 183 184 Args: 185 186 X: {array-like}, shape = [n_samples, n_features] 187 Training vectors, where n_samples is the number 188 of samples and n_features is the number of features. 189 190 y: array-like, shape = [n_samples] 191 Target values. 192 193 **kwargs: additional parameters to be passed to 194 self.cook_training_set or self.obj.fit 195 196 Returns: 197 198 self: object 199 200 """ 201 202 assert mx.is_factor(y), "y must contain only integers" 203 204 # training 205 self.n_classes = len(np.unique(y)) 206 207 base_learner = CustomClassifier( 208 self.obj, 209 n_hidden_features=self.n_hidden_features, 210 activation_name=self.activation_name, 211 a=self.a, 212 nodes_sim=self.nodes_sim, 213 bias=self.bias, 214 dropout=self.dropout, 215 direct_link=self.direct_link, 216 n_clusters=self.n_clusters, 217 type_clust=self.type_clust, 218 type_scaling=self.type_scaling, 219 col_sample=self.col_sample, 220 row_sample=self.row_sample, 221 seed=self.seed, 222 ) 223 224 # 1 - Sequential training ----- 225 226 if self.n_jobs is None: 227 self.voter_ = rbagloop_classification( 228 base_learner, X, y, self.n_estimators, self.verbose, self.seed 229 ) 230 231 self.n_estimators = len(self.voter_) 232 233 return self 234 235 # 2 - Parallel training ----- 236 # buggy 237 # if self.n_jobs is not None: 238 def fit_estimators(m): 239 base_learner__ = pickle.loads(pickle.dumps(base_learner, -1)) 240 base_learner__.set_params(seed=self.seed + m * 1000) 241 base_learner__.fit(X, y, **kwargs) 242 return base_learner__ 243 244 if self.verbose == 1: 245 voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")( 246 delayed(fit_estimators)(m) 247 for m in tqdm(range(self.n_estimators)) 248 ) 249 else: 250 voters_list = Parallel(n_jobs=self.n_jobs, prefer="threads")( 251 delayed(fit_estimators)(m) for m in range(self.n_estimators) 252 ) 253 254 self.voter_ = {idx: elt for idx, elt in enumerate(voters_list)} 255 256 self.n_estimators = len(self.voter_) 257 258 return self
Fit Random 'Bagging' model to training data (X, y).
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
260 def predict(self, X, weights=None, **kwargs): 261 """Predict test data X. 262 263 Args: 264 265 X: {array-like}, shape = [n_samples, n_features] 266 Training vectors, where n_samples is the number 267 of samples and n_features is the number of features. 268 269 **kwargs: additional parameters to be passed to 270 self.cook_test_set 271 272 Returns: 273 274 model predictions: {array-like} 275 276 """ 277 return self.predict_proba(X, weights, **kwargs).argmax(axis=1)
Predict test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
279 def predict_proba(self, X, weights=None, **kwargs): 280 """Predict probabilities for test data X. 281 282 Args: 283 284 X: {array-like}, shape = [n_samples, n_features] 285 Training vectors, where n_samples is the number 286 of samples and n_features is the number of features. 287 288 **kwargs: additional parameters to be passed to 289 self.cook_test_set 290 291 Returns: 292 293 probability estimates for test data: {array-like} 294 295 """ 296 297 def calculate_probas(voter, weights=None, verbose=None): 298 ensemble_proba = 0 299 300 n_iter = len(voter) 301 302 assert n_iter > 0, "no estimator found in `RandomBag` ensemble" 303 304 if weights is None: 305 for idx, elt in voter.items(): 306 try: 307 ensemble_proba += elt.predict_proba(X) 308 309 # if verbose == 1: 310 # pbar.update(idx) 311 312 except: 313 continue 314 315 # if verbose == 1: 316 # pbar.update(n_iter) 317 318 return ensemble_proba / n_iter 319 320 # if weights is not None: 321 for idx, elt in voter.items(): 322 ensemble_proba += weights[idx] * elt.predict_proba(X) 323 324 # if verbose == 1: 325 # pbar.update(idx) 326 327 # if verbose == 1: 328 # pbar.update(n_iter) 329 330 return ensemble_proba 331 332 # end calculate_probas ---- 333 334 if self.n_jobs is None: 335 # if self.verbose == 1: 336 # pbar = Progbar(self.n_estimators) 337 338 if weights is None: 339 return calculate_probas(self.voter_, verbose=self.verbose) 340 341 # if weights is not None: 342 self.weights = weights 343 344 return calculate_probas(self.voter_, weights, verbose=self.verbose) 345 346 # if self.n_jobs is not None: 347 def predict_estimator(m): 348 try: 349 return self.voter_[m].predict_proba(X) 350 except: 351 pass 352 353 if self.verbose == 1: 354 preds = Parallel(n_jobs=self.n_jobs, prefer="threads")( 355 delayed(predict_estimator)(m) 356 for m in tqdm(range(self.n_estimators)) 357 ) 358 359 else: 360 preds = Parallel(n_jobs=self.n_jobs, prefer="threads")( 361 delayed(predict_estimator)(m) for m in range(self.n_estimators) 362 ) 363 364 ensemble_proba = 0 365 366 if weights is None: 367 for i in range(self.n_estimators): 368 ensemble_proba += preds[i] 369 370 return ensemble_proba / self.n_estimators 371 372 for i in range(self.n_estimators): 373 ensemble_proba += weights[i] * preds[i] 374 375 return ensemble_proba
Predict probabilities for test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
probability estimates for test data: {array-like}
377 def score(self, X, y, weights=None, scoring=None, **kwargs): 378 """ Score the model on test set features X and response y. 379 380 Args: 381 382 X: {array-like}, shape = [n_samples, n_features] 383 Training vectors, where n_samples is the number 384 of samples and n_features is the number of features 385 386 y: array-like, shape = [n_samples] 387 Target values 388 389 scoring: str 390 must be in ('explained_variance', 'neg_mean_absolute_error', \ 391 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 392 'neg_median_absolute_error', 'r2') 393 394 **kwargs: additional parameters to be passed to scoring functions 395 396 Returns: 397 398 model scores: {array-like} 399 400 """ 401 402 preds = self.predict(X, weights, **kwargs) 403 404 if scoring is None: 405 scoring = "accuracy" 406 407 # check inputs 408 assert scoring in ( 409 "accuracy", 410 "average_precision", 411 "brier_score_loss", 412 "f1", 413 "f1_micro", 414 "f1_macro", 415 "f1_weighted", 416 "f1_samples", 417 "neg_log_loss", 418 "precision", 419 "recall", 420 "roc_auc", 421 ), "'scoring' should be in ('accuracy', 'average_precision', \ 422 'brier_score_loss', 'f1', 'f1_micro', \ 423 'f1_macro', 'f1_weighted', 'f1_samples', \ 424 'neg_log_loss', 'precision', 'recall', \ 425 'roc_auc')" 426 427 scoring_options = { 428 "accuracy": skm2.accuracy_score, 429 "average_precision": skm2.average_precision_score, 430 "brier_score_loss": skm2.brier_score_loss, 431 "f1": skm2.f1_score, 432 "f1_micro": skm2.f1_score, 433 "f1_macro": skm2.f1_score, 434 "f1_weighted": skm2.f1_score, 435 "f1_samples": skm2.f1_score, 436 "neg_log_loss": skm2.log_loss, 437 "precision": skm2.precision_score, 438 "recall": skm2.recall_score, 439 "roc_auc": skm2.roc_auc_score, 440 } 441 442 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.randombag.bag.RandomBag
- obj
- n_estimators
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
21class Ridge2Regressor(Ridge2, RegressorMixin): 22 """Ridge regression with 2 regularization parameters derived from class Ridge 23 24 Parameters: 25 26 n_hidden_features: int 27 number of nodes in the hidden layer 28 29 activation_name: str 30 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 31 32 a: float 33 hyperparameter for 'prelu' or 'elu' activation function 34 35 nodes_sim: str 36 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 37 'uniform' 38 39 bias: boolean 40 indicates if the hidden layer contains a bias term (True) or not 41 (False) 42 43 dropout: float 44 regularization parameter; (random) percentage of nodes dropped out 45 of the training 46 47 n_clusters: int 48 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: 49 no clustering) 50 51 cluster_encode: bool 52 defines how the variable containing clusters is treated (default is one-hot) 53 if `False`, then labels are used, without one-hot encoding 54 55 type_clust: str 56 type of clustering method: currently k-means ('kmeans') or Gaussian 57 Mixture Model ('gmm') 58 59 type_scaling: a tuple of 3 strings 60 scaling methods for inputs, hidden layer, and clustering respectively 61 (and when relevant). 62 Currently available: standardization ('std') or MinMax scaling ('minmax') 63 64 lambda1: float 65 regularization parameter on direct link 66 67 lambda2: float 68 regularization parameter on hidden layer 69 70 seed: int 71 reproducibility seed for nodes_sim=='uniform' 72 73 backend: str 74 'cpu' or 'gpu' or 'tpu' 75 76 Attributes: 77 78 beta_: {array-like} 79 regression coefficients 80 81 y_mean_: float 82 average response 83 84 """ 85 86 # construct the object ----- 87 88 def __init__( 89 self, 90 n_hidden_features=5, 91 activation_name="relu", 92 a=0.01, 93 nodes_sim="sobol", 94 bias=True, 95 dropout=0, 96 n_clusters=2, 97 cluster_encode=True, 98 type_clust="kmeans", 99 type_scaling=("std", "std", "std"), 100 lambda1=0.1, 101 lambda2=0.1, 102 seed=123, 103 backend="cpu", 104 ): 105 super().__init__( 106 n_hidden_features=n_hidden_features, 107 activation_name=activation_name, 108 a=a, 109 nodes_sim=nodes_sim, 110 bias=bias, 111 dropout=dropout, 112 n_clusters=n_clusters, 113 cluster_encode=cluster_encode, 114 type_clust=type_clust, 115 type_scaling=type_scaling, 116 lambda1=lambda1, 117 lambda2=lambda2, 118 seed=seed, 119 backend=backend, 120 ) 121 122 self.type_fit = "regression" 123 124 def fit(self, X, y, **kwargs): 125 """Fit Ridge model to training data (X, y). 126 127 Args: 128 129 X: {array-like}, shape = [n_samples, n_features] 130 Training vectors, where n_samples is the number 131 of samples and n_features is the number of features. 132 133 y: array-like, shape = [n_samples] 134 Target values. 135 136 **kwargs: additional parameters to be passed to 137 self.cook_training_set or self.obj.fit 138 139 Returns: 140 141 self: object 142 143 """ 144 145 sys_platform = platform.system() 146 147 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 148 149 n_X, p_X = X.shape 150 n_Z, p_Z = scaled_Z.shape 151 152 if self.n_clusters > 0: 153 if self.encode_clusters == True: 154 n_features = p_X + self.n_clusters 155 else: 156 n_features = p_X + 1 157 else: 158 n_features = p_X 159 160 X_ = scaled_Z[:, 0:n_features] 161 Phi_X_ = scaled_Z[:, n_features:p_Z] 162 163 B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag( 164 np.repeat(1, n_features) 165 ) 166 C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend) 167 D = mo.crossprod( 168 x=Phi_X_, backend=self.backend 169 ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1])) 170 171 if sys_platform in ("Linux", "Darwin"): 172 B_inv = pinv(B) if self.backend == "cpu" else jpinv(B) 173 else: 174 B_inv = pinv(B) 175 176 W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend) 177 S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend) 178 179 if sys_platform in ("Linux", "Darwin"): 180 S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat) 181 else: 182 S_inv = pinv(S_mat) 183 184 Y = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend) 185 inv = mo.rbind( 186 mo.cbind( 187 x=B_inv + mo.crossprod(x=W, y=Y, backend=self.backend), 188 y=-np.transpose(Y), 189 backend=self.backend, 190 ), 191 mo.cbind(x=-Y, y=S_inv, backend=self.backend), 192 backend=self.backend, 193 ) 194 195 self.beta_ = mo.safe_sparse_dot( 196 a=inv, 197 b=mo.crossprod(x=scaled_Z, y=centered_y, backend=self.backend), 198 backend=self.backend, 199 ) 200 201 return self 202 203 def predict(self, X, **kwargs): 204 """Predict test data X. 205 206 Args: 207 208 X: {array-like}, shape = [n_samples, n_features] 209 Training vectors, where n_samples is the number 210 of samples and n_features is the number of features. 211 212 **kwargs: additional parameters to be passed to 213 self.cook_test_set 214 215 Returns: 216 217 model predictions: {array-like} 218 219 """ 220 221 if len(X.shape) == 1: 222 n_features = X.shape[0] 223 new_X = mo.rbind( 224 x=X.reshape(1, n_features), 225 y=np.ones(n_features).reshape(1, n_features), 226 backend=self.backend, 227 ) 228 229 return ( 230 self.y_mean_ 231 + mo.safe_sparse_dot( 232 a=self.cook_test_set(new_X, **kwargs), 233 b=self.beta_, 234 backend=self.backend, 235 ) 236 )[0] 237 238 return self.y_mean_ + mo.safe_sparse_dot( 239 a=self.cook_test_set(X, **kwargs), 240 b=self.beta_, 241 backend=self.backend, 242 ) 243 244 def score(self, X, y, scoring=None, **kwargs): 245 """ Score the model on test set features X and response y. 246 247 Args: 248 249 X: {array-like}, shape = [n_samples, n_features] 250 Training vectors, where n_samples is the number 251 of samples and n_features is the number of features 252 253 y: array-like, shape = [n_samples] 254 Target values 255 256 scoring: str 257 must be in ('explained_variance', 'neg_mean_absolute_error', \ 258 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 259 'neg_median_absolute_error', 'r2') 260 261 **kwargs: additional parameters to be passed to scoring functions 262 263 Returns: 264 265 model scores: {array-like} 266 267 """ 268 269 preds = self.predict(X) 270 271 if type(preds) == tuple: # if there are std. devs in the predictions 272 preds = preds[0] 273 274 if scoring is None: 275 scoring = "neg_mean_squared_error" 276 277 # check inputs 278 assert scoring in ( 279 "explained_variance", 280 "neg_mean_absolute_error", 281 "neg_mean_squared_error", 282 "neg_mean_squared_log_error", 283 "neg_median_absolute_error", 284 "r2", 285 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 286 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 287 'neg_median_absolute_error', 'r2')" 288 289 scoring_options = { 290 "explained_variance": skm.explained_variance_score, 291 "neg_mean_absolute_error": skm.median_absolute_error, 292 "neg_mean_squared_error": skm.mean_squared_error, 293 "neg_mean_squared_log_error": skm.mean_squared_log_error, 294 "neg_median_absolute_error": skm.median_absolute_error, 295 "r2": skm.r2_score, 296 } 297 298 return scoring_options[scoring](y, preds, **kwargs)
Ridge regression with 2 regularization parameters derived from class Ridge
Parameters:
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not
(False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
lambda1: float
regularization parameter on direct link
lambda2: float
regularization parameter on hidden layer
seed: int
reproducibility seed for nodes_sim=='uniform'
backend: str
'cpu' or 'gpu' or 'tpu'
Attributes:
beta_: {array-like}
regression coefficients
y_mean_: float
average response
88 def __init__( 89 self, 90 n_hidden_features=5, 91 activation_name="relu", 92 a=0.01, 93 nodes_sim="sobol", 94 bias=True, 95 dropout=0, 96 n_clusters=2, 97 cluster_encode=True, 98 type_clust="kmeans", 99 type_scaling=("std", "std", "std"), 100 lambda1=0.1, 101 lambda2=0.1, 102 seed=123, 103 backend="cpu", 104 ): 105 super().__init__( 106 n_hidden_features=n_hidden_features, 107 activation_name=activation_name, 108 a=a, 109 nodes_sim=nodes_sim, 110 bias=bias, 111 dropout=dropout, 112 n_clusters=n_clusters, 113 cluster_encode=cluster_encode, 114 type_clust=type_clust, 115 type_scaling=type_scaling, 116 lambda1=lambda1, 117 lambda2=lambda2, 118 seed=seed, 119 backend=backend, 120 ) 121 122 self.type_fit = "regression"
124 def fit(self, X, y, **kwargs): 125 """Fit Ridge model to training data (X, y). 126 127 Args: 128 129 X: {array-like}, shape = [n_samples, n_features] 130 Training vectors, where n_samples is the number 131 of samples and n_features is the number of features. 132 133 y: array-like, shape = [n_samples] 134 Target values. 135 136 **kwargs: additional parameters to be passed to 137 self.cook_training_set or self.obj.fit 138 139 Returns: 140 141 self: object 142 143 """ 144 145 sys_platform = platform.system() 146 147 centered_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 148 149 n_X, p_X = X.shape 150 n_Z, p_Z = scaled_Z.shape 151 152 if self.n_clusters > 0: 153 if self.encode_clusters == True: 154 n_features = p_X + self.n_clusters 155 else: 156 n_features = p_X + 1 157 else: 158 n_features = p_X 159 160 X_ = scaled_Z[:, 0:n_features] 161 Phi_X_ = scaled_Z[:, n_features:p_Z] 162 163 B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag( 164 np.repeat(1, n_features) 165 ) 166 C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend) 167 D = mo.crossprod( 168 x=Phi_X_, backend=self.backend 169 ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1])) 170 171 if sys_platform in ("Linux", "Darwin"): 172 B_inv = pinv(B) if self.backend == "cpu" else jpinv(B) 173 else: 174 B_inv = pinv(B) 175 176 W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend) 177 S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend) 178 179 if sys_platform in ("Linux", "Darwin"): 180 S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat) 181 else: 182 S_inv = pinv(S_mat) 183 184 Y = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend) 185 inv = mo.rbind( 186 mo.cbind( 187 x=B_inv + mo.crossprod(x=W, y=Y, backend=self.backend), 188 y=-np.transpose(Y), 189 backend=self.backend, 190 ), 191 mo.cbind(x=-Y, y=S_inv, backend=self.backend), 192 backend=self.backend, 193 ) 194 195 self.beta_ = mo.safe_sparse_dot( 196 a=inv, 197 b=mo.crossprod(x=scaled_Z, y=centered_y, backend=self.backend), 198 backend=self.backend, 199 ) 200 201 return self
Fit Ridge model to training data (X, y).
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
203 def predict(self, X, **kwargs): 204 """Predict test data X. 205 206 Args: 207 208 X: {array-like}, shape = [n_samples, n_features] 209 Training vectors, where n_samples is the number 210 of samples and n_features is the number of features. 211 212 **kwargs: additional parameters to be passed to 213 self.cook_test_set 214 215 Returns: 216 217 model predictions: {array-like} 218 219 """ 220 221 if len(X.shape) == 1: 222 n_features = X.shape[0] 223 new_X = mo.rbind( 224 x=X.reshape(1, n_features), 225 y=np.ones(n_features).reshape(1, n_features), 226 backend=self.backend, 227 ) 228 229 return ( 230 self.y_mean_ 231 + mo.safe_sparse_dot( 232 a=self.cook_test_set(new_X, **kwargs), 233 b=self.beta_, 234 backend=self.backend, 235 ) 236 )[0] 237 238 return self.y_mean_ + mo.safe_sparse_dot( 239 a=self.cook_test_set(X, **kwargs), 240 b=self.beta_, 241 backend=self.backend, 242 )
Predict test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
244 def score(self, X, y, scoring=None, **kwargs): 245 """ Score the model on test set features X and response y. 246 247 Args: 248 249 X: {array-like}, shape = [n_samples, n_features] 250 Training vectors, where n_samples is the number 251 of samples and n_features is the number of features 252 253 y: array-like, shape = [n_samples] 254 Target values 255 256 scoring: str 257 must be in ('explained_variance', 'neg_mean_absolute_error', \ 258 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 259 'neg_median_absolute_error', 'r2') 260 261 **kwargs: additional parameters to be passed to scoring functions 262 263 Returns: 264 265 model scores: {array-like} 266 267 """ 268 269 preds = self.predict(X) 270 271 if type(preds) == tuple: # if there are std. devs in the predictions 272 preds = preds[0] 273 274 if scoring is None: 275 scoring = "neg_mean_squared_error" 276 277 # check inputs 278 assert scoring in ( 279 "explained_variance", 280 "neg_mean_absolute_error", 281 "neg_mean_squared_error", 282 "neg_mean_squared_log_error", 283 "neg_median_absolute_error", 284 "r2", 285 ), "'scoring' should be in ('explained_variance', 'neg_mean_absolute_error', \ 286 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 287 'neg_median_absolute_error', 'r2')" 288 289 scoring_options = { 290 "explained_variance": skm.explained_variance_score, 291 "neg_mean_absolute_error": skm.median_absolute_error, 292 "neg_mean_squared_error": skm.mean_squared_error, 293 "neg_mean_squared_log_error": skm.mean_squared_log_error, 294 "neg_median_absolute_error": skm.median_absolute_error, 295 "r2": skm.r2_score, 296 } 297 298 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.ridge2.ridge2.Ridge2
- lambda1
- lambda2
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
18class Ridge2Classifier(Ridge2, ClassifierMixin): 19 """Multinomial logit classification with 2 regularization parameters 20 21 Parameters: 22 23 n_hidden_features: int 24 number of nodes in the hidden layer 25 26 activation_name: str 27 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 28 29 a: float 30 hyperparameter for 'prelu' or 'elu' activation function 31 32 nodes_sim: str 33 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 34 'uniform' 35 36 bias: boolean 37 indicates if the hidden layer contains a bias term (True) or not 38 (False) 39 40 dropout: float 41 regularization parameter; (random) percentage of nodes dropped out 42 of the training 43 44 direct_link: boolean 45 indicates if the original predictors are included (True) in model's 46 fitting or not (False) 47 48 n_clusters: int 49 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: 50 no clustering) 51 52 cluster_encode: bool 53 defines how the variable containing clusters is treated (default is one-hot) 54 if `False`, then labels are used, without one-hot encoding 55 56 type_clust: str 57 type of clustering method: currently k-means ('kmeans') or Gaussian 58 Mixture Model ('gmm') 59 60 type_scaling: a tuple of 3 strings 61 scaling methods for inputs, hidden layer, and clustering respectively 62 (and when relevant). 63 Currently available: standardization ('std') or MinMax scaling ('minmax') 64 65 lambda1: float 66 regularization parameter on direct link 67 68 lambda2: float 69 regularization parameter on hidden layer 70 71 seed: int 72 reproducibility seed for nodes_sim=='uniform' 73 74 backend: str 75 "cpu" or "gpu" or "tpu" 76 77 Attributes: 78 79 beta_: {array-like} 80 regression coefficients 81 82 83 Examples: 84 85 See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/ridge_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/ridge_classification.py) 86 87 ```python 88 import nnetsauce as ns 89 import numpy as np 90 from sklearn.datasets import load_breast_cancer 91 from sklearn.model_selection import train_test_split 92 from time import time 93 94 95 breast_cancer = load_breast_cancer() 96 X = breast_cancer.data 97 y = breast_cancer.target 98 99 # split data into training test and test set 100 np.random.seed(123) 101 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) 102 103 # create the model with nnetsauce 104 fit_obj = ns.Ridge2Classifier(lambda1 = 6.90185578e+04, 105 lambda2 = 3.17392781e+02, 106 n_hidden_features=95, 107 n_clusters=2, 108 dropout = 3.62817383e-01, 109 type_clust = "gmm") 110 111 # fit the model on training set 112 start = time() 113 fit_obj.fit(X_train, y_train) 114 print(f"Elapsed {time() - start}") 115 116 # get the accuracy on test set 117 start = time() 118 print(fit_obj.score(X_test, y_test)) 119 print(f"Elapsed {time() - start}") 120 121 # get area under the curve on test set (auc) 122 print(fit_obj.score(X_test, y_test, scoring="roc_auc")) 123 ``` 124 125 126 """ 127 128 # construct the object ----- 129 130 def __init__( 131 self, 132 n_hidden_features=5, 133 activation_name="relu", 134 a=0.01, 135 nodes_sim="sobol", 136 bias=True, 137 dropout=0, 138 direct_link=True, 139 n_clusters=2, 140 cluster_encode=True, 141 type_clust="kmeans", 142 type_scaling=("std", "std", "std"), 143 lambda1=0.1, 144 lambda2=0.1, 145 seed=123, 146 backend="cpu", 147 ): 148 super().__init__( 149 n_hidden_features=n_hidden_features, 150 activation_name=activation_name, 151 a=a, 152 nodes_sim=nodes_sim, 153 bias=bias, 154 dropout=dropout, 155 direct_link=direct_link, 156 n_clusters=n_clusters, 157 cluster_encode=cluster_encode, 158 type_clust=type_clust, 159 type_scaling=type_scaling, 160 lambda1=lambda1, 161 lambda2=lambda2, 162 seed=seed, 163 backend=backend, 164 ) 165 166 self.type_fit = "classification" 167 168 def loglik(self, X, Y, **kwargs): 169 """Log-likelihood for training data (X, Y). 170 171 Args: 172 173 X: {array-like}, shape = [n_samples, n_features] 174 Training vectors, where n_samples is the number 175 of samples and n_features is the number of features. 176 177 Y: array-like, shape = [n_samples] 178 One-hot encode target values. 179 180 **kwargs: additional parameters to be passed to 181 self.cook_training_set or self.obj.fit 182 183 Returns: 184 185 """ 186 187 def loglik_grad_hess(Y, X, B, XB, hessian=True, **kwargs): 188 # nobs, n_classes 189 n, K = Y.shape 190 191 # total number of covariates 192 p = X.shape[1] 193 194 # initial number of covariates 195 init_p = p - self.n_hidden_features 196 197 max_double = 709.0 198 XB[XB > max_double] = max_double 199 exp_XB = np.exp(XB) 200 probs = exp_XB / exp_XB.sum(axis=1)[:, None] 201 202 # gradient ----- 203 # (Y - p) -> (n, K) 204 # X -> (n, p) 205 # (K, n) %*% (n, p) -> (K, p) 206 if hessian is False: 207 grad = ( 208 -mo.safe_sparse_dot( 209 a=(Y - probs).T, b=X, backend=self.backend 210 ) 211 / n 212 ) 213 grad += self.lambda1 * B[0:init_p, :].sum(axis=0)[:, None] 214 grad += self.lambda2 * B[init_p:p, :].sum(axis=0)[:, None] 215 216 return grad.flatten() 217 218 # hessian ----- 219 if hessian is True: 220 Kp = K * p 221 hess = np.zeros((Kp, Kp), float) 222 for k1 in range(K): 223 x_index = range(k1 * p, (k1 + 1) * p) 224 for k2 in range(k1, K): 225 y_index = range(k2 * p, (k2 + 1) * p) 226 H_sub = ( 227 -mo.safe_sparse_dot( 228 a=X.T, 229 b=(probs[:, k1] * probs[:, k2])[:, None] * X, 230 backend=self.backend, 231 ) 232 / n 233 ) # do not store 234 hess[np.ix_(x_index, y_index)] = hess[ 235 np.ix_(y_index, x_index) 236 ] = H_sub 237 238 return hess + (self.lambda1 + self.lambda2) * np.identity(Kp) 239 240 # total number of covariates 241 p = X.shape[1] 242 243 # initial number of covariates 244 init_p = p - self.n_hidden_features 245 246 # log-likelihood (1st return) 247 def loglik_func(x): 248 # (p, K) 249 B = x.reshape(Y.shape[1], p).T 250 251 # (n, K) 252 XB = mo.safe_sparse_dot(X, B, backend=self.backend) 253 254 res = -(np.sum(Y * XB, axis=1) - logsumexp(XB)).mean() 255 256 res += ( 257 0.5 258 * self.lambda1 259 * mo.squared_norm(B[0:init_p, :], backend=self.backend) 260 ) 261 res += ( 262 0.5 263 * self.lambda2 264 * mo.squared_norm(B[init_p:p, :], backend=self.backend) 265 ) 266 267 return res 268 269 # gradient of log-likelihood 270 def grad_func(x): 271 # (p, K) 272 B = x.reshape(Y.shape[1], p).T 273 274 return loglik_grad_hess( 275 Y=Y, 276 X=X, 277 B=B, 278 XB=mo.safe_sparse_dot(X, B, backend=self.backend), 279 hessian=False, 280 **kwargs 281 ) 282 283 # hessian of log-likelihood 284 def hessian_func(x): 285 # (p, K) 286 B = x.reshape(Y.shape[1], p).T 287 288 return loglik_grad_hess( 289 Y=Y, 290 X=X, 291 B=B, 292 XB=mo.safe_sparse_dot(X, B, backend=self.backend), 293 hessian=True, 294 **kwargs 295 ) 296 297 return loglik_func, grad_func, hessian_func 298 299 # newton-cg 300 # L-BFGS-B 301 def fit(self, X, y, solver="L-BFGS-B", **kwargs): 302 """Fit Ridge model to training data (X, y). 303 304 for beta: regression coeffs (beta11, ..., beta1p, ..., betaK1, ..., betaKp) 305 for K classes and p covariates. 306 307 Args: 308 309 X: {array-like}, shape = [n_samples, n_features] 310 Training vectors, where n_samples is the number 311 of samples and n_features is the number of features. 312 313 y: array-like, shape = [n_samples] 314 Target values. 315 316 **kwargs: additional parameters to be passed to 317 self.cook_training_set or self.obj.fit 318 319 Returns: 320 321 self: object 322 323 """ 324 325 assert mx.is_factor(y), "y must contain only integers" 326 327 output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 328 329 self.n_classes = len(np.unique(y)) 330 331 Y = mo.one_hot_encode2(output_y, self.n_classes) 332 333 # optimize for beta, minimize self.loglik (maximize loglik) ----- 334 loglik_func, grad_func, hessian_func = self.loglik(X=scaled_Z, Y=Y) 335 336 if solver == "L-BFGS-B": 337 self.beta_ = minimize( 338 fun=loglik_func, 339 x0=np.zeros(scaled_Z.shape[1] * self.n_classes), 340 jac=grad_func, 341 method=solver, 342 ).x 343 344 if solver in ("Newton-CG", "trust-ncg"): 345 self.beta_ = minimize( 346 fun=loglik_func, 347 x0=np.zeros(scaled_Z.shape[1] * self.n_classes), 348 jac=grad_func, 349 hess=hessian_func, 350 method=solver, 351 ).x 352 353 return self 354 355 def predict(self, X, **kwargs): 356 """Predict test data X. 357 358 Args: 359 360 X: {array-like}, shape = [n_samples, n_features] 361 Training vectors, where n_samples is the number 362 of samples and n_features is the number of features. 363 364 **kwargs: additional parameters to be passed to 365 self.cook_test_set 366 367 Returns: 368 369 model predictions: {array-like} 370 """ 371 372 return np.argmax(self.predict_proba(X, **kwargs), axis=1) 373 374 def predict_proba(self, X, **kwargs): 375 """Predict probabilities for test data X. 376 377 Args: 378 379 X: {array-like}, shape = [n_samples, n_features] 380 Training vectors, where n_samples is the number 381 of samples and n_features is the number of features. 382 383 **kwargs: additional parameters to be passed to 384 self.cook_test_set 385 386 Returns: 387 388 probability estimates for test data: {array-like} 389 390 """ 391 if len(X.shape) == 1: 392 n_features = X.shape[0] 393 new_X = mo.rbind( 394 X.reshape(1, n_features), 395 np.ones(n_features).reshape(1, n_features), 396 ) 397 398 Z = self.cook_test_set(new_X, **kwargs) 399 400 else: 401 Z = self.cook_test_set(X, **kwargs) 402 403 ZB = mo.safe_sparse_dot( 404 a=Z, 405 b=self.beta_.reshape( 406 self.n_classes, 407 X.shape[1] + self.n_hidden_features + self.n_clusters, 408 ).T, 409 backend=self.backend, 410 ) 411 412 exp_ZB = np.exp(ZB) 413 414 return exp_ZB / exp_ZB.sum(axis=1)[:, None] 415 416 def score(self, X, y, scoring=None, **kwargs): 417 """ Score the model on test set features X and response y. 418 419 Args: 420 421 X: {array-like}, shape = [n_samples, n_features] 422 Training vectors, where n_samples is the number 423 of samples and n_features is the number of features 424 425 y: array-like, shape = [n_samples] 426 Target values 427 428 scoring: str 429 must be in ('explained_variance', 'neg_mean_absolute_error', \ 430 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 431 'neg_median_absolute_error', 'r2') 432 433 **kwargs: additional parameters to be passed to scoring functions 434 435 Returns: 436 437 model scores: {array-like} 438 439 """ 440 441 preds = self.predict(X) 442 443 if scoring is None: 444 scoring = "accuracy" 445 446 # check inputs 447 assert scoring in ( 448 "accuracy", 449 "average_precision", 450 "brier_score_loss", 451 "f1", 452 "f1_micro", 453 "f1_macro", 454 "f1_weighted", 455 "f1_samples", 456 "neg_log_loss", 457 "precision", 458 "recall", 459 "roc_auc", 460 ), "'scoring' should be in ('accuracy', 'average_precision', \ 461 'brier_score_loss', 'f1', 'f1_micro', \ 462 'f1_macro', 'f1_weighted', 'f1_samples', \ 463 'neg_log_loss', 'precision', 'recall', \ 464 'roc_auc')" 465 466 scoring_options = { 467 "accuracy": skm2.accuracy_score, 468 "average_precision": skm2.average_precision_score, 469 "brier_score_loss": skm2.brier_score_loss, 470 "f1": skm2.f1_score, 471 "f1_micro": skm2.f1_score, 472 "f1_macro": skm2.f1_score, 473 "f1_weighted": skm2.f1_score, 474 "f1_samples": skm2.f1_score, 475 "neg_log_loss": skm2.log_loss, 476 "precision": skm2.precision_score, 477 "recall": skm2.recall_score, 478 "roc_auc": skm2.roc_auc_score, 479 } 480 481 return scoring_options[scoring](y, preds, **kwargs)
Multinomial logit classification with 2 regularization parameters
Parameters:
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not
(False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
direct_link: boolean
indicates if the original predictors are included (True) in model's
fitting or not (False)
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
lambda1: float
regularization parameter on direct link
lambda2: float
regularization parameter on hidden layer
seed: int
reproducibility seed for nodes_sim=='uniform'
backend: str
"cpu" or "gpu" or "tpu"
Attributes:
beta_: {array-like}
regression coefficients
Examples:
See also https://github.com/Techtonique/nnetsauce/blob/master/examples/ridge_classification.py
import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from time import time
breast_cancer = load_breast_cancer()
X = breast_cancer.data
y = breast_cancer.target
# split data into training test and test set
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
# create the model with nnetsauce
fit_obj = ns.Ridge2Classifier(lambda1 = 6.90185578e+04,
lambda2 = 3.17392781e+02,
n_hidden_features=95,
n_clusters=2,
dropout = 3.62817383e-01,
type_clust = "gmm")
# fit the model on training set
start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")
# get the accuracy on test set
start = time()
print(fit_obj.score(X_test, y_test))
print(f"Elapsed {time() - start}")
# get area under the curve on test set (auc)
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
130 def __init__( 131 self, 132 n_hidden_features=5, 133 activation_name="relu", 134 a=0.01, 135 nodes_sim="sobol", 136 bias=True, 137 dropout=0, 138 direct_link=True, 139 n_clusters=2, 140 cluster_encode=True, 141 type_clust="kmeans", 142 type_scaling=("std", "std", "std"), 143 lambda1=0.1, 144 lambda2=0.1, 145 seed=123, 146 backend="cpu", 147 ): 148 super().__init__( 149 n_hidden_features=n_hidden_features, 150 activation_name=activation_name, 151 a=a, 152 nodes_sim=nodes_sim, 153 bias=bias, 154 dropout=dropout, 155 direct_link=direct_link, 156 n_clusters=n_clusters, 157 cluster_encode=cluster_encode, 158 type_clust=type_clust, 159 type_scaling=type_scaling, 160 lambda1=lambda1, 161 lambda2=lambda2, 162 seed=seed, 163 backend=backend, 164 ) 165 166 self.type_fit = "classification"
168 def loglik(self, X, Y, **kwargs): 169 """Log-likelihood for training data (X, Y). 170 171 Args: 172 173 X: {array-like}, shape = [n_samples, n_features] 174 Training vectors, where n_samples is the number 175 of samples and n_features is the number of features. 176 177 Y: array-like, shape = [n_samples] 178 One-hot encode target values. 179 180 **kwargs: additional parameters to be passed to 181 self.cook_training_set or self.obj.fit 182 183 Returns: 184 185 """ 186 187 def loglik_grad_hess(Y, X, B, XB, hessian=True, **kwargs): 188 # nobs, n_classes 189 n, K = Y.shape 190 191 # total number of covariates 192 p = X.shape[1] 193 194 # initial number of covariates 195 init_p = p - self.n_hidden_features 196 197 max_double = 709.0 198 XB[XB > max_double] = max_double 199 exp_XB = np.exp(XB) 200 probs = exp_XB / exp_XB.sum(axis=1)[:, None] 201 202 # gradient ----- 203 # (Y - p) -> (n, K) 204 # X -> (n, p) 205 # (K, n) %*% (n, p) -> (K, p) 206 if hessian is False: 207 grad = ( 208 -mo.safe_sparse_dot( 209 a=(Y - probs).T, b=X, backend=self.backend 210 ) 211 / n 212 ) 213 grad += self.lambda1 * B[0:init_p, :].sum(axis=0)[:, None] 214 grad += self.lambda2 * B[init_p:p, :].sum(axis=0)[:, None] 215 216 return grad.flatten() 217 218 # hessian ----- 219 if hessian is True: 220 Kp = K * p 221 hess = np.zeros((Kp, Kp), float) 222 for k1 in range(K): 223 x_index = range(k1 * p, (k1 + 1) * p) 224 for k2 in range(k1, K): 225 y_index = range(k2 * p, (k2 + 1) * p) 226 H_sub = ( 227 -mo.safe_sparse_dot( 228 a=X.T, 229 b=(probs[:, k1] * probs[:, k2])[:, None] * X, 230 backend=self.backend, 231 ) 232 / n 233 ) # do not store 234 hess[np.ix_(x_index, y_index)] = hess[ 235 np.ix_(y_index, x_index) 236 ] = H_sub 237 238 return hess + (self.lambda1 + self.lambda2) * np.identity(Kp) 239 240 # total number of covariates 241 p = X.shape[1] 242 243 # initial number of covariates 244 init_p = p - self.n_hidden_features 245 246 # log-likelihood (1st return) 247 def loglik_func(x): 248 # (p, K) 249 B = x.reshape(Y.shape[1], p).T 250 251 # (n, K) 252 XB = mo.safe_sparse_dot(X, B, backend=self.backend) 253 254 res = -(np.sum(Y * XB, axis=1) - logsumexp(XB)).mean() 255 256 res += ( 257 0.5 258 * self.lambda1 259 * mo.squared_norm(B[0:init_p, :], backend=self.backend) 260 ) 261 res += ( 262 0.5 263 * self.lambda2 264 * mo.squared_norm(B[init_p:p, :], backend=self.backend) 265 ) 266 267 return res 268 269 # gradient of log-likelihood 270 def grad_func(x): 271 # (p, K) 272 B = x.reshape(Y.shape[1], p).T 273 274 return loglik_grad_hess( 275 Y=Y, 276 X=X, 277 B=B, 278 XB=mo.safe_sparse_dot(X, B, backend=self.backend), 279 hessian=False, 280 **kwargs 281 ) 282 283 # hessian of log-likelihood 284 def hessian_func(x): 285 # (p, K) 286 B = x.reshape(Y.shape[1], p).T 287 288 return loglik_grad_hess( 289 Y=Y, 290 X=X, 291 B=B, 292 XB=mo.safe_sparse_dot(X, B, backend=self.backend), 293 hessian=True, 294 **kwargs 295 ) 296 297 return loglik_func, grad_func, hessian_func
Log-likelihood for training data (X, Y).
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
Y: array-like, shape = [n_samples]
One-hot encode target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
301 def fit(self, X, y, solver="L-BFGS-B", **kwargs): 302 """Fit Ridge model to training data (X, y). 303 304 for beta: regression coeffs (beta11, ..., beta1p, ..., betaK1, ..., betaKp) 305 for K classes and p covariates. 306 307 Args: 308 309 X: {array-like}, shape = [n_samples, n_features] 310 Training vectors, where n_samples is the number 311 of samples and n_features is the number of features. 312 313 y: array-like, shape = [n_samples] 314 Target values. 315 316 **kwargs: additional parameters to be passed to 317 self.cook_training_set or self.obj.fit 318 319 Returns: 320 321 self: object 322 323 """ 324 325 assert mx.is_factor(y), "y must contain only integers" 326 327 output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 328 329 self.n_classes = len(np.unique(y)) 330 331 Y = mo.one_hot_encode2(output_y, self.n_classes) 332 333 # optimize for beta, minimize self.loglik (maximize loglik) ----- 334 loglik_func, grad_func, hessian_func = self.loglik(X=scaled_Z, Y=Y) 335 336 if solver == "L-BFGS-B": 337 self.beta_ = minimize( 338 fun=loglik_func, 339 x0=np.zeros(scaled_Z.shape[1] * self.n_classes), 340 jac=grad_func, 341 method=solver, 342 ).x 343 344 if solver in ("Newton-CG", "trust-ncg"): 345 self.beta_ = minimize( 346 fun=loglik_func, 347 x0=np.zeros(scaled_Z.shape[1] * self.n_classes), 348 jac=grad_func, 349 hess=hessian_func, 350 method=solver, 351 ).x 352 353 return self
Fit Ridge model to training data (X, y).
for beta: regression coeffs (beta11, ..., beta1p, ..., betaK1, ..., betaKp) for K classes and p covariates.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
355 def predict(self, X, **kwargs): 356 """Predict test data X. 357 358 Args: 359 360 X: {array-like}, shape = [n_samples, n_features] 361 Training vectors, where n_samples is the number 362 of samples and n_features is the number of features. 363 364 **kwargs: additional parameters to be passed to 365 self.cook_test_set 366 367 Returns: 368 369 model predictions: {array-like} 370 """ 371 372 return np.argmax(self.predict_proba(X, **kwargs), axis=1)
Predict test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
374 def predict_proba(self, X, **kwargs): 375 """Predict probabilities for test data X. 376 377 Args: 378 379 X: {array-like}, shape = [n_samples, n_features] 380 Training vectors, where n_samples is the number 381 of samples and n_features is the number of features. 382 383 **kwargs: additional parameters to be passed to 384 self.cook_test_set 385 386 Returns: 387 388 probability estimates for test data: {array-like} 389 390 """ 391 if len(X.shape) == 1: 392 n_features = X.shape[0] 393 new_X = mo.rbind( 394 X.reshape(1, n_features), 395 np.ones(n_features).reshape(1, n_features), 396 ) 397 398 Z = self.cook_test_set(new_X, **kwargs) 399 400 else: 401 Z = self.cook_test_set(X, **kwargs) 402 403 ZB = mo.safe_sparse_dot( 404 a=Z, 405 b=self.beta_.reshape( 406 self.n_classes, 407 X.shape[1] + self.n_hidden_features + self.n_clusters, 408 ).T, 409 backend=self.backend, 410 ) 411 412 exp_ZB = np.exp(ZB) 413 414 return exp_ZB / exp_ZB.sum(axis=1)[:, None]
Predict probabilities for test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
probability estimates for test data: {array-like}
416 def score(self, X, y, scoring=None, **kwargs): 417 """ Score the model on test set features X and response y. 418 419 Args: 420 421 X: {array-like}, shape = [n_samples, n_features] 422 Training vectors, where n_samples is the number 423 of samples and n_features is the number of features 424 425 y: array-like, shape = [n_samples] 426 Target values 427 428 scoring: str 429 must be in ('explained_variance', 'neg_mean_absolute_error', \ 430 'neg_mean_squared_error', 'neg_mean_squared_log_error', \ 431 'neg_median_absolute_error', 'r2') 432 433 **kwargs: additional parameters to be passed to scoring functions 434 435 Returns: 436 437 model scores: {array-like} 438 439 """ 440 441 preds = self.predict(X) 442 443 if scoring is None: 444 scoring = "accuracy" 445 446 # check inputs 447 assert scoring in ( 448 "accuracy", 449 "average_precision", 450 "brier_score_loss", 451 "f1", 452 "f1_micro", 453 "f1_macro", 454 "f1_weighted", 455 "f1_samples", 456 "neg_log_loss", 457 "precision", 458 "recall", 459 "roc_auc", 460 ), "'scoring' should be in ('accuracy', 'average_precision', \ 461 'brier_score_loss', 'f1', 'f1_micro', \ 462 'f1_macro', 'f1_weighted', 'f1_samples', \ 463 'neg_log_loss', 'precision', 'recall', \ 464 'roc_auc')" 465 466 scoring_options = { 467 "accuracy": skm2.accuracy_score, 468 "average_precision": skm2.average_precision_score, 469 "brier_score_loss": skm2.brier_score_loss, 470 "f1": skm2.f1_score, 471 "f1_micro": skm2.f1_score, 472 "f1_macro": skm2.f1_score, 473 "f1_weighted": skm2.f1_score, 474 "f1_samples": skm2.f1_score, 475 "neg_log_loss": skm2.log_loss, 476 "precision": skm2.precision_score, 477 "recall": skm2.recall_score, 478 "roc_auc": skm2.roc_auc_score, 479 } 480 481 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set features X and response y.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features
y: array-like, shape = [n_samples]
Target values
scoring: str
must be in ('explained_variance', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2')
**kwargs: additional parameters to be passed to scoring functions
Returns:
model scores: {array-like}
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.ridge2.ridge2.Ridge2
- lambda1
- lambda2
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
21class Ridge2MultitaskClassifier(Ridge2, ClassifierMixin): 22 """Multitask Ridge classification with 2 regularization parameters 23 24 Parameters: 25 26 n_hidden_features: int 27 number of nodes in the hidden layer 28 29 activation_name: str 30 activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu' 31 32 a: float 33 hyperparameter for 'prelu' or 'elu' activation function 34 35 nodes_sim: str 36 type of simulation for the nodes: 'sobol', 'hammersley', 'halton', 37 'uniform' 38 39 bias: boolean 40 indicates if the hidden layer contains a bias term (True) or not 41 (False) 42 43 dropout: float 44 regularization parameter; (random) percentage of nodes dropped out 45 of the training 46 47 n_clusters: int 48 number of clusters for 'kmeans' or 'gmm' clustering (could be 0: 49 no clustering) 50 51 cluster_encode: bool 52 defines how the variable containing clusters is treated (default is one-hot) 53 if `False`, then labels are used, without one-hot encoding 54 55 type_clust: str 56 type of clustering method: currently k-means ('kmeans') or Gaussian 57 Mixture Model ('gmm') 58 59 type_scaling: a tuple of 3 strings 60 scaling methods for inputs, hidden layer, and clustering respectively 61 (and when relevant). 62 Currently available: standardization ('std') or MinMax scaling ('minmax') 63 64 lambda1: float 65 regularization parameter on direct link 66 67 lambda2: float 68 regularization parameter on hidden layer 69 70 seed: int 71 reproducibility seed for nodes_sim=='uniform' 72 73 backend: str 74 "cpu" or "gpu" or "tpu" 75 76 Attributes: 77 78 beta_: {array-like} 79 regression coefficients 80 81 Examples: 82 83 See also [https://github.com/Techtonique/nnetsauce/blob/master/examples/ridgemtask_classification.py](https://github.com/Techtonique/nnetsauce/blob/master/examples/ridgemtask_classification.py) 84 85 ```python 86 import nnetsauce as ns 87 import numpy as np 88 from sklearn.datasets import load_breast_cancer 89 from sklearn.model_selection import train_test_split 90 from sklearn import metrics 91 from time import time 92 93 breast_cancer = load_breast_cancer() 94 Z = breast_cancer.data 95 t = breast_cancer.target 96 np.random.seed(123) 97 X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2) 98 99 fit_obj = ns.Ridge2MultitaskClassifier(n_hidden_features=int(9.83730469e+01), 100 dropout=4.31054687e-01, 101 n_clusters=int(1.71484375e+00), 102 lambda1=1.24023438e+01, lambda2=7.30263672e+03) 103 104 start = time() 105 fit_obj.fit(X_train, y_train) 106 print(f"Elapsed {time() - start}") 107 108 print(fit_obj.score(X_test, y_test)) 109 print(fit_obj.score(X_test, y_test, scoring="roc_auc")) 110 111 start = time() 112 preds = fit_obj.predict(X_test) 113 print(f"Elapsed {time() - start}") 114 print(metrics.classification_report(preds, y_test)) 115 ``` 116 117 """ 118 119 # construct the object ----- 120 121 def __init__( 122 self, 123 n_hidden_features=5, 124 activation_name="relu", 125 a=0.01, 126 nodes_sim="sobol", 127 bias=True, 128 dropout=0, 129 n_clusters=2, 130 cluster_encode=True, 131 type_clust="kmeans", 132 type_scaling=("std", "std", "std"), 133 lambda1=0.1, 134 lambda2=0.1, 135 seed=123, 136 backend="cpu", 137 ): 138 super().__init__( 139 n_hidden_features=n_hidden_features, 140 activation_name=activation_name, 141 a=a, 142 nodes_sim=nodes_sim, 143 bias=bias, 144 dropout=dropout, 145 n_clusters=n_clusters, 146 cluster_encode=cluster_encode, 147 type_clust=type_clust, 148 type_scaling=type_scaling, 149 lambda1=lambda1, 150 lambda2=lambda2, 151 seed=seed, 152 backend=backend, 153 ) 154 155 self.type_fit = "regression" 156 157 def fit(self, X, y, **kwargs): 158 """Fit Ridge model to training data (X, y). 159 160 Args: 161 162 X: {array-like}, shape = [n_samples, n_features] 163 Training vectors, where n_samples is the number 164 of samples and n_features is the number of features. 165 166 y: array-like, shape = [n_samples] 167 Target values. 168 169 **kwargs: additional parameters to be passed to 170 self.cook_training_set or self.obj.fit 171 172 Returns: 173 174 self: object 175 176 """ 177 178 sys_platform = platform.system() 179 180 assert mx.is_factor(y), "y must contain only integers" 181 182 output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 183 184 n_X, p_X = X.shape 185 n_Z, p_Z = scaled_Z.shape 186 187 self.n_classes = len(np.unique(y)) 188 189 # multitask response 190 Y = mo.one_hot_encode2(output_y, self.n_classes) 191 192 if self.n_clusters > 0: 193 if self.encode_clusters == True: 194 n_features = p_X + self.n_clusters 195 else: 196 n_features = p_X + 1 197 else: 198 n_features = p_X 199 200 X_ = scaled_Z[:, 0:n_features] 201 Phi_X_ = scaled_Z[:, n_features:p_Z] 202 203 B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag( 204 np.repeat(1, X_.shape[1]) 205 ) 206 C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend) 207 D = mo.crossprod( 208 x=Phi_X_, backend=self.backend 209 ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1])) 210 211 if sys_platform in ("Linux", "Darwin"): 212 B_inv = pinv(B) if self.backend == "cpu" else jpinv(B) 213 else: 214 B_inv = pinv(B) 215 216 W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend) 217 S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend) 218 219 if sys_platform in ("Linux", "Darwin"): 220 S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat) 221 else: 222 S_inv = pinv(S_mat) 223 224 Y2 = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend) 225 inv = mo.rbind( 226 mo.cbind( 227 x=B_inv + mo.crossprod(x=W, y=Y2, backend=self.backend), 228 y=-np.transpose(Y2), 229 backend=self.backend, 230 ), 231 mo.cbind(x=-Y2, y=S_inv, backend=self.backend), 232 backend=self.backend, 233 ) 234 235 self.beta_ = mo.safe_sparse_dot( 236 a=inv, 237 b=mo.crossprod(x=scaled_Z, y=Y, backend=self.backend), 238 backend=self.backend, 239 ) 240 241 return self 242 243 def predict(self, X, **kwargs): 244 """Predict test data X. 245 246 Args: 247 248 X: {array-like}, shape = [n_samples, n_features] 249 Training vectors, where n_samples is the number 250 of samples and n_features is the number of features. 251 252 **kwargs: additional parameters to be passed to 253 self.cook_test_set 254 255 Returns: 256 257 model predictions: {array-like} 258 259 """ 260 261 return np.argmax(self.predict_proba(X, **kwargs), axis=1) 262 263 def predict_proba(self, X, **kwargs): 264 """Predict probabilities for test data X. 265 266 Args: 267 268 X: {array-like}, shape = [n_samples, n_features] 269 Training vectors, where n_samples is the number 270 of samples and n_features is the number of features. 271 272 **kwargs: additional parameters to be passed to 273 self.cook_test_set 274 275 Returns: 276 277 probability estimates for test data: {array-like} 278 279 """ 280 281 if len(X.shape) == 1: 282 n_features = X.shape[0] 283 new_X = mo.rbind( 284 x=X.reshape(1, n_features), 285 y=np.ones(n_features).reshape(1, n_features), 286 backend=self.backend, 287 ) 288 289 Z = self.cook_test_set(new_X, **kwargs) 290 291 else: 292 Z = self.cook_test_set(X, **kwargs) 293 294 ZB = mo.safe_sparse_dot(a=Z, b=self.beta_, backend=self.backend) 295 296 exp_ZB = np.exp(ZB) 297 298 return exp_ZB / exp_ZB.sum(axis=1)[:, None] 299 300 def score(self, X, y, scoring=None, **kwargs): 301 """Score the model on test set covariates X and response y.""" 302 303 preds = self.predict(X) 304 305 if scoring is None: 306 scoring = "accuracy" 307 308 # check inputs 309 assert scoring in ( 310 "accuracy", 311 "average_precision", 312 "brier_score_loss", 313 "f1", 314 "f1_micro", 315 "f1_macro", 316 "f1_weighted", 317 "f1_samples", 318 "neg_log_loss", 319 "precision", 320 "recall", 321 "roc_auc", 322 ), "'scoring' should be in ('accuracy', 'average_precision', \ 323 'brier_score_loss', 'f1', 'f1_micro', \ 324 'f1_macro', 'f1_weighted', 'f1_samples', \ 325 'neg_log_loss', 'precision', 'recall', \ 326 'roc_auc')" 327 328 scoring_options = { 329 "accuracy": skm2.accuracy_score, 330 "average_precision": skm2.average_precision_score, 331 "brier_score_loss": skm2.brier_score_loss, 332 "f1": skm2.f1_score, 333 "f1_micro": skm2.f1_score, 334 "f1_macro": skm2.f1_score, 335 "f1_weighted": skm2.f1_score, 336 "f1_samples": skm2.f1_score, 337 "neg_log_loss": skm2.log_loss, 338 "precision": skm2.precision_score, 339 "recall": skm2.recall_score, 340 "roc_auc": skm2.roc_auc_score, 341 } 342 343 return scoring_options[scoring](y, preds, **kwargs)
Multitask Ridge classification with 2 regularization parameters
Parameters:
n_hidden_features: int
number of nodes in the hidden layer
activation_name: str
activation function: 'relu', 'tanh', 'sigmoid', 'prelu' or 'elu'
a: float
hyperparameter for 'prelu' or 'elu' activation function
nodes_sim: str
type of simulation for the nodes: 'sobol', 'hammersley', 'halton',
'uniform'
bias: boolean
indicates if the hidden layer contains a bias term (True) or not
(False)
dropout: float
regularization parameter; (random) percentage of nodes dropped out
of the training
n_clusters: int
number of clusters for 'kmeans' or 'gmm' clustering (could be 0:
no clustering)
cluster_encode: bool
defines how the variable containing clusters is treated (default is one-hot)
if `False`, then labels are used, without one-hot encoding
type_clust: str
type of clustering method: currently k-means ('kmeans') or Gaussian
Mixture Model ('gmm')
type_scaling: a tuple of 3 strings
scaling methods for inputs, hidden layer, and clustering respectively
(and when relevant).
Currently available: standardization ('std') or MinMax scaling ('minmax')
lambda1: float
regularization parameter on direct link
lambda2: float
regularization parameter on hidden layer
seed: int
reproducibility seed for nodes_sim=='uniform'
backend: str
"cpu" or "gpu" or "tpu"
Attributes:
beta_: {array-like}
regression coefficients
Examples:
See also https://github.com/Techtonique/nnetsauce/blob/master/examples/ridgemtask_classification.py
import nnetsauce as ns
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn import metrics
from time import time
breast_cancer = load_breast_cancer()
Z = breast_cancer.data
t = breast_cancer.target
np.random.seed(123)
X_train, X_test, y_train, y_test = train_test_split(Z, t, test_size=0.2)
fit_obj = ns.Ridge2MultitaskClassifier(n_hidden_features=int(9.83730469e+01),
dropout=4.31054687e-01,
n_clusters=int(1.71484375e+00),
lambda1=1.24023438e+01, lambda2=7.30263672e+03)
start = time()
fit_obj.fit(X_train, y_train)
print(f"Elapsed {time() - start}")
print(fit_obj.score(X_test, y_test))
print(fit_obj.score(X_test, y_test, scoring="roc_auc"))
start = time()
preds = fit_obj.predict(X_test)
print(f"Elapsed {time() - start}")
print(metrics.classification_report(preds, y_test))
121 def __init__( 122 self, 123 n_hidden_features=5, 124 activation_name="relu", 125 a=0.01, 126 nodes_sim="sobol", 127 bias=True, 128 dropout=0, 129 n_clusters=2, 130 cluster_encode=True, 131 type_clust="kmeans", 132 type_scaling=("std", "std", "std"), 133 lambda1=0.1, 134 lambda2=0.1, 135 seed=123, 136 backend="cpu", 137 ): 138 super().__init__( 139 n_hidden_features=n_hidden_features, 140 activation_name=activation_name, 141 a=a, 142 nodes_sim=nodes_sim, 143 bias=bias, 144 dropout=dropout, 145 n_clusters=n_clusters, 146 cluster_encode=cluster_encode, 147 type_clust=type_clust, 148 type_scaling=type_scaling, 149 lambda1=lambda1, 150 lambda2=lambda2, 151 seed=seed, 152 backend=backend, 153 ) 154 155 self.type_fit = "regression"
157 def fit(self, X, y, **kwargs): 158 """Fit Ridge model to training data (X, y). 159 160 Args: 161 162 X: {array-like}, shape = [n_samples, n_features] 163 Training vectors, where n_samples is the number 164 of samples and n_features is the number of features. 165 166 y: array-like, shape = [n_samples] 167 Target values. 168 169 **kwargs: additional parameters to be passed to 170 self.cook_training_set or self.obj.fit 171 172 Returns: 173 174 self: object 175 176 """ 177 178 sys_platform = platform.system() 179 180 assert mx.is_factor(y), "y must contain only integers" 181 182 output_y, scaled_Z = self.cook_training_set(y=y, X=X, **kwargs) 183 184 n_X, p_X = X.shape 185 n_Z, p_Z = scaled_Z.shape 186 187 self.n_classes = len(np.unique(y)) 188 189 # multitask response 190 Y = mo.one_hot_encode2(output_y, self.n_classes) 191 192 if self.n_clusters > 0: 193 if self.encode_clusters == True: 194 n_features = p_X + self.n_clusters 195 else: 196 n_features = p_X + 1 197 else: 198 n_features = p_X 199 200 X_ = scaled_Z[:, 0:n_features] 201 Phi_X_ = scaled_Z[:, n_features:p_Z] 202 203 B = mo.crossprod(x=X_, backend=self.backend) + self.lambda1 * np.diag( 204 np.repeat(1, X_.shape[1]) 205 ) 206 C = mo.crossprod(x=Phi_X_, y=X_, backend=self.backend) 207 D = mo.crossprod( 208 x=Phi_X_, backend=self.backend 209 ) + self.lambda2 * np.diag(np.repeat(1, Phi_X_.shape[1])) 210 211 if sys_platform in ("Linux", "Darwin"): 212 B_inv = pinv(B) if self.backend == "cpu" else jpinv(B) 213 else: 214 B_inv = pinv(B) 215 216 W = mo.safe_sparse_dot(a=C, b=B_inv, backend=self.backend) 217 S_mat = D - mo.tcrossprod(x=W, y=C, backend=self.backend) 218 219 if sys_platform in ("Linux", "Darwin"): 220 S_inv = pinv(S_mat) if self.backend == "cpu" else jpinv(S_mat) 221 else: 222 S_inv = pinv(S_mat) 223 224 Y2 = mo.safe_sparse_dot(a=S_inv, b=W, backend=self.backend) 225 inv = mo.rbind( 226 mo.cbind( 227 x=B_inv + mo.crossprod(x=W, y=Y2, backend=self.backend), 228 y=-np.transpose(Y2), 229 backend=self.backend, 230 ), 231 mo.cbind(x=-Y2, y=S_inv, backend=self.backend), 232 backend=self.backend, 233 ) 234 235 self.beta_ = mo.safe_sparse_dot( 236 a=inv, 237 b=mo.crossprod(x=scaled_Z, y=Y, backend=self.backend), 238 backend=self.backend, 239 ) 240 241 return self
Fit Ridge model to training data (X, y).
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
y: array-like, shape = [n_samples]
Target values.
**kwargs: additional parameters to be passed to
self.cook_training_set or self.obj.fit
Returns:
self: object
243 def predict(self, X, **kwargs): 244 """Predict test data X. 245 246 Args: 247 248 X: {array-like}, shape = [n_samples, n_features] 249 Training vectors, where n_samples is the number 250 of samples and n_features is the number of features. 251 252 **kwargs: additional parameters to be passed to 253 self.cook_test_set 254 255 Returns: 256 257 model predictions: {array-like} 258 259 """ 260 261 return np.argmax(self.predict_proba(X, **kwargs), axis=1)
Predict test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
model predictions: {array-like}
263 def predict_proba(self, X, **kwargs): 264 """Predict probabilities for test data X. 265 266 Args: 267 268 X: {array-like}, shape = [n_samples, n_features] 269 Training vectors, where n_samples is the number 270 of samples and n_features is the number of features. 271 272 **kwargs: additional parameters to be passed to 273 self.cook_test_set 274 275 Returns: 276 277 probability estimates for test data: {array-like} 278 279 """ 280 281 if len(X.shape) == 1: 282 n_features = X.shape[0] 283 new_X = mo.rbind( 284 x=X.reshape(1, n_features), 285 y=np.ones(n_features).reshape(1, n_features), 286 backend=self.backend, 287 ) 288 289 Z = self.cook_test_set(new_X, **kwargs) 290 291 else: 292 Z = self.cook_test_set(X, **kwargs) 293 294 ZB = mo.safe_sparse_dot(a=Z, b=self.beta_, backend=self.backend) 295 296 exp_ZB = np.exp(ZB) 297 298 return exp_ZB / exp_ZB.sum(axis=1)[:, None]
Predict probabilities for test data X.
Args:
X: {array-like}, shape = [n_samples, n_features]
Training vectors, where n_samples is the number
of samples and n_features is the number of features.
**kwargs: additional parameters to be passed to
self.cook_test_set
Returns:
probability estimates for test data: {array-like}
300 def score(self, X, y, scoring=None, **kwargs): 301 """Score the model on test set covariates X and response y.""" 302 303 preds = self.predict(X) 304 305 if scoring is None: 306 scoring = "accuracy" 307 308 # check inputs 309 assert scoring in ( 310 "accuracy", 311 "average_precision", 312 "brier_score_loss", 313 "f1", 314 "f1_micro", 315 "f1_macro", 316 "f1_weighted", 317 "f1_samples", 318 "neg_log_loss", 319 "precision", 320 "recall", 321 "roc_auc", 322 ), "'scoring' should be in ('accuracy', 'average_precision', \ 323 'brier_score_loss', 'f1', 'f1_micro', \ 324 'f1_macro', 'f1_weighted', 'f1_samples', \ 325 'neg_log_loss', 'precision', 'recall', \ 326 'roc_auc')" 327 328 scoring_options = { 329 "accuracy": skm2.accuracy_score, 330 "average_precision": skm2.average_precision_score, 331 "brier_score_loss": skm2.brier_score_loss, 332 "f1": skm2.f1_score, 333 "f1_micro": skm2.f1_score, 334 "f1_macro": skm2.f1_score, 335 "f1_weighted": skm2.f1_score, 336 "f1_samples": skm2.f1_score, 337 "neg_log_loss": skm2.log_loss, 338 "precision": skm2.precision_score, 339 "recall": skm2.recall_score, 340 "roc_auc": skm2.roc_auc_score, 341 } 342 343 return scoring_options[scoring](y, preds, **kwargs)
Score the model on test set covariates X and response y.
A descriptor for request methods.
New in version 1.3.
Parameters
name : str
The name of the method for which the request function should be
created, e.g. "fit" would create a set_fit_request function.
keys : list of str
A list of strings which are accepted parameters by the created
function, e.g. ["sample_weight"] if the corresponding method
accepts it as a metadata.
validate_keys : bool, default=True Whether to check if the requested parameters fit the actual parameters of the method.
Notes
This class is a descriptor 1 and uses PEP-362 to set the signature of the returned function 2.
References
Inherited Members
- nnetsauce.ridge2.ridge2.Ridge2
- lambda1
- lambda2
- Base
- activation_name
- a
- nodes_sim
- bias
- seed
- backend
- dropout
- direct_link
- cluster_encode
- type_clust
- type_scaling
- col_sample
- row_sample
- n_clusters
- subsampler_
- index_col_
- index_row_
- clustering_obj_
- clustering_scaler_
- nn_scaler_
- scaler_
- encoder_
- W_
- X_
- y_
- y_mean_
- beta_
- activation_func
- encode_clusters
- create_layer
- cook_training_set
- cook_test_set
- sklearn.base.BaseEstimator
- get_params
- set_params
- sklearn.utils._metadata_requests._MetadataRequester
- get_metadata_routing
5class SubSampler: 6 """Subsampling class. 7 8 Attributes: 9 10 y: array-like, shape = [n_samples] 11 Target values. 12 13 row_sample: double 14 subsampling fraction 15 16 n_samples: int 17 subsampling by using the number of rows 18 19 seed: int 20 reproductibility seed 21 22 n_jobs: int 23 number of jobs to run in parallel 24 25 verbose: bool 26 print progress messages and bars 27 28 Returns: 29 30 indices of subsampled y 31 32 """ 33 34 def __init__( 35 self, 36 y, 37 row_sample=0.8, 38 n_samples=None, 39 seed=123, 40 n_jobs=None, 41 verbose=False, 42 ): 43 self.y = y 44 self.row_sample = row_sample 45 self.n_samples = n_samples 46 self.seed = seed 47 self.indices = None 48 self.n_jobs = n_jobs 49 self.verbose = verbose 50 51 def subsample(self): 52 self.indices = dosubsample( 53 self.y, 54 self.row_sample, 55 self.n_samples, 56 self.seed, 57 self.n_jobs, 58 self.verbose, 59 ) 60 return self.indices
Subsampling class.
Attributes:
y: array-like, shape = [n_samples] Target values.
row_sample: double subsampling fraction
n_samples: int subsampling by using the number of rows
seed: int reproductibility seed
n_jobs: int number of jobs to run in parallel
verbose: bool print progress messages and bars
Returns:
indices of subsampled y